From d080dc24a8f5e225550ba860eb474816df6c42a2 Mon Sep 17 00:00:00 2001 From: evgyur <120411712+evgyur@users.noreply.github.com> Date: Sun, 26 Jul 2026 05:39:26 +0300 Subject: [PATCH 01/69] fix(gateway): skip topic recovery for non-DM messages --- gateway/platforms/base.py | 15 +++-- .../gateway/test_active_session_text_merge.py | 57 +++++++++++++++++++ 2 files changed, 67 insertions(+), 5 deletions(-) diff --git a/gateway/platforms/base.py b/gateway/platforms/base.py index d7654ff6c147..de60322718bb 100644 --- a/gateway/platforms/base.py +++ b/gateway/platforms/base.py @@ -5545,11 +5545,16 @@ async def handle_message(self, event: MessageEvent) -> None: coerce_plaintext_gateway_command(event) - # Rewrite ``event.source.thread_id`` via the installed recovery hook - # (Telegram DM topic mode) so the session key, guard checks, and - # downstream delivery all agree on the same lane. - # Offloaded: the sync hook must not block the loop. - await asyncio.to_thread(self._apply_topic_recovery, event) + # Telegram topic recovery only applies to private DM topic lanes. Do + # not submit a no-op check for group/forum/channel traffic to the + # shared default executor: a busy pool would delay message dispatch. + needs_topic_recovery = ( + getattr(self, "_topic_recovery_fn", None) is not None + and event.source.platform == Platform.TELEGRAM + and event.source.chat_type == "dm" + ) + if needs_topic_recovery: + await asyncio.to_thread(self._apply_topic_recovery, event) session_key = build_session_key( event.source, diff --git a/tests/gateway/test_active_session_text_merge.py b/tests/gateway/test_active_session_text_merge.py index e3dd819b2884..ac8df747c650 100644 --- a/tests/gateway/test_active_session_text_merge.py +++ b/tests/gateway/test_active_session_text_merge.py @@ -115,6 +115,63 @@ def _debounced_event(adapter: BasePlatformAdapter, session_key: str) -> MessageE return adapter._text_debounce[session_key].event +@pytest.mark.asyncio +async def test_non_dm_message_does_not_wait_for_topic_recovery_executor(monkeypatch): + """Group messages must not queue behind the shared thread pool. + + Topic recovery only applies to Telegram DM topic mode. Offloading that + no-op check for every group message makes ingress wait behind unrelated + blocking jobs when the default executor is saturated. + """ + adapter = _make_adapter() + recovery = MagicMock(return_value=None) + adapter.set_topic_recovery_fn(recovery) + executor_called = False + never_release = asyncio.Event() + + async def _blocked_to_thread(*args, **kwargs): + nonlocal executor_called + executor_called = True + await never_release.wait() + + monkeypatch.setattr(asyncio, "to_thread", _blocked_to_thread) + + await asyncio.wait_for( + adapter.handle_message(_make_event("/status", chat_type="group")), + timeout=1.0, + ) + await asyncio.sleep(0) + + assert executor_called is False + recovery.assert_not_called() + + +@pytest.mark.asyncio +async def test_dm_topic_recovery_stays_offloaded(monkeypatch): + """Real Telegram DM topic recovery must still run outside the event loop.""" + adapter = _make_adapter() + recovery = MagicMock(return_value="topic-222") + adapter.set_topic_recovery_fn(recovery) + offloaded = False + + async def _inline_to_thread(func, *args, **kwargs): + nonlocal offloaded + offloaded = True + return func(*args, **kwargs) + + monkeypatch.setattr(asyncio, "to_thread", _inline_to_thread) + event = _make_event("hello", chat_type="dm", thread_id="1") + original_source = event.source + + await adapter.handle_message(event) + await asyncio.sleep(0) + + assert offloaded is True + assert recovery.call_count == 1 + assert recovery.call_args.args[0] is original_source + assert event.source.thread_id == "topic-222" + + @pytest.mark.asyncio async def test_rapid_text_followups_accumulate_instead_of_replacing(): """Rapid TEXT follow-ups must all survive in the pending event.""" From f1da9d0d66a311df442e647aa7e803a8c9f6f38c Mon Sep 17 00:00:00 2001 From: LeonSGP43 Date: Fri, 10 Jul 2026 19:35:03 -0700 Subject: [PATCH 02/69] fix(docker): skip redundant stage2 chown walks --- docker/stage2-hook.sh | 14 ++++++++++---- tests/tools/test_stage2_hook_symlink_chown.py | 8 ++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/docker/stage2-hook.sh b/docker/stage2-hook.sh index 05474ca927b4..df7a1c83b500 100755 --- a/docker/stage2-hook.sh +++ b/docker/stage2-hook.sh @@ -220,6 +220,11 @@ chown_hermes_tree() { echo "[stage2] Warning: chown $target failed (rootless container?) — continuing" } +tree_has_non_hermes_owner() { + target="$1" + find "$target" \( ! -user hermes -o ! -group hermes \) -print -quit 2>/dev/null | grep -q . +} + needs_chown=false if [ "$(stat -c %u "$HERMES_HOME" 2>/dev/null)" != "$actual_hermes_uid" ]; then needs_chown=true @@ -243,7 +248,7 @@ if [ "$needs_chown" = true ]; then # created and managed exclusively by hermes (see the s6-setuidgid mkdir # -p block below for the canonical list). for sub in cron sessions logs hooks memories skills skins plans workspace home profiles pairing platforms/pairing lazy-packages; do - if [ -e "$HERMES_HOME/$sub" ]; then + if [ -e "$HERMES_HOME/$sub" ] && tree_has_non_hermes_owner "$HERMES_HOME/$sub"; then chown_hermes_tree "$HERMES_HOME/$sub" fi done @@ -273,9 +278,10 @@ fi # are invoked via `docker exec hermes …` (which defaults # to root unless `-u` is passed), and that breaks the cont-init # reconciler (02-reconcile-profiles) which runs as hermes and walks -# the profiles dir. Idempotent; skipped on rootless containers where -# chown would fail. -if [ -d "$HERMES_HOME/profiles" ]; then +# the profiles dir. Skip the recursive walk when the tree is already +# owned correctly so warm boots do not rescan huge profile caches. +# Idempotent; skipped on rootless containers where chown would fail. +if [ -d "$HERMES_HOME/profiles" ] && tree_has_non_hermes_owner "$HERMES_HOME/profiles"; then chown_hermes_tree "$HERMES_HOME/profiles" fi diff --git a/tests/tools/test_stage2_hook_symlink_chown.py b/tests/tools/test_stage2_hook_symlink_chown.py index cc8d0312d9ac..8c2a95233f92 100644 --- a/tests/tools/test_stage2_hook_symlink_chown.py +++ b/tests/tools/test_stage2_hook_symlink_chown.py @@ -114,3 +114,11 @@ def test_stage2_skips_top_level_chown_for_symlinked_hermes_home( stage2_text: str, ) -> None: assert 'refuse_symlinked_path "chown" "$HERMES_HOME"' in stage2_text + + +def test_stage2_skips_recursive_repairs_when_tree_is_already_owned( + stage2_text: str, +) -> None: + assert "tree_has_non_hermes_owner() {" in stage2_text + assert 'if [ -e "$HERMES_HOME/$sub" ] && tree_has_non_hermes_owner "$HERMES_HOME/$sub"; then' in stage2_text + assert 'if [ -d "$HERMES_HOME/profiles" ] && tree_has_non_hermes_owner "$HERMES_HOME/profiles"; then' in stage2_text From 4983c576b1ed118d56ddf98feed468f4b3a7042c Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Sun, 2 Aug 2026 21:49:58 +0530 Subject: [PATCH 03/69] fix(docker): gate the remaining every-boot chown walks (cron, pairing) Whole-bug-class follow-up to the profiles/ gate: cron/, platforms/ pairing, and legacy pairing/ ran chown_hermes_tree unconditionally on every boot with the identical warm-boot cost profile. Same tree_has_non_hermes_owner gate; find evaluates the top directory first and -quits on the first mismatch, so a mis-owned tree short-circuits in O(1) while a clean tree pays one read-only walk instead of a full chown -R inode rewrite. --- docker/stage2-hook.sh | 15 +++++++++------ tests/tools/test_stage2_hook_symlink_chown.py | 4 ++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/docker/stage2-hook.sh b/docker/stage2-hook.sh index df7a1c83b500..899c8e86ac98 100755 --- a/docker/stage2-hook.sh +++ b/docker/stage2-hook.sh @@ -288,8 +288,10 @@ fi # Always reset ownership of $HERMES_HOME/cron on every boot for the same # docker-exec/root-write reason as profiles/. The cron scheduler state # (jobs.json) must stay readable by the unprivileged hermes runtime even -# after root-context maintenance commands or scheduler writes. -if [ -d "$HERMES_HOME/cron" ]; then +# after root-context maintenance commands or scheduler writes. Skip the +# recursive walk when the tree is already owned correctly (same warm-boot +# gate as profiles/). +if [ -d "$HERMES_HOME/cron" ] && tree_has_non_hermes_owner "$HERMES_HOME/cron"; then chown_hermes_tree "$HERMES_HOME/cron" fi @@ -315,13 +317,14 @@ fi # silently leaving the approved user unauthorized (#10270). The targeted # data-volume chown above only runs when the top-level $HERMES_HOME is # mis-owned, so warm boots skip it — this block makes a container restart -# self-heal. Tiny directory (a handful of small JSON files), so the cost -# is negligible. -if [ -d "$HERMES_HOME/platforms/pairing" ]; then +# self-heal. Tiny directory (a handful of small JSON files), so even the +# ownership pre-scan is negligible; gated for consistency with profiles/ +# and cron/. +if [ -d "$HERMES_HOME/platforms/pairing" ] && tree_has_non_hermes_owner "$HERMES_HOME/platforms/pairing"; then chown_hermes_tree "$HERMES_HOME/platforms/pairing" fi # Legacy location (pre-consolidated layout). -if [ -d "$HERMES_HOME/pairing" ]; then +if [ -d "$HERMES_HOME/pairing" ] && tree_has_non_hermes_owner "$HERMES_HOME/pairing"; then chown_hermes_tree "$HERMES_HOME/pairing" fi diff --git a/tests/tools/test_stage2_hook_symlink_chown.py b/tests/tools/test_stage2_hook_symlink_chown.py index 8c2a95233f92..fd68d78def7f 100644 --- a/tests/tools/test_stage2_hook_symlink_chown.py +++ b/tests/tools/test_stage2_hook_symlink_chown.py @@ -122,3 +122,7 @@ def test_stage2_skips_recursive_repairs_when_tree_is_already_owned( assert "tree_has_non_hermes_owner() {" in stage2_text assert 'if [ -e "$HERMES_HOME/$sub" ] && tree_has_non_hermes_owner "$HERMES_HOME/$sub"; then' in stage2_text assert 'if [ -d "$HERMES_HOME/profiles" ] && tree_has_non_hermes_owner "$HERMES_HOME/profiles"; then' in stage2_text + # Sibling every-boot chown blocks carry the same warm-boot gate. + assert 'if [ -d "$HERMES_HOME/cron" ] && tree_has_non_hermes_owner "$HERMES_HOME/cron"; then' in stage2_text + assert 'if [ -d "$HERMES_HOME/platforms/pairing" ] && tree_has_non_hermes_owner "$HERMES_HOME/platforms/pairing"; then' in stage2_text + assert 'if [ -d "$HERMES_HOME/pairing" ] && tree_has_non_hermes_owner "$HERMES_HOME/pairing"; then' in stage2_text From 0d9f17ffd433f9661ac57661b9ad9915d670f657 Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Sun, 2 Aug 2026 22:01:54 +0530 Subject: [PATCH 04/69] chore: add contributor email mapping for ZachariahChu --- contributors/emails/1762459322@qq.com | 1 + 1 file changed, 1 insertion(+) create mode 100644 contributors/emails/1762459322@qq.com diff --git a/contributors/emails/1762459322@qq.com b/contributors/emails/1762459322@qq.com new file mode 100644 index 000000000000..6decf7d09654 --- /dev/null +++ b/contributors/emails/1762459322@qq.com @@ -0,0 +1 @@ +ZachariahChu From 5d1875fe3adce2130344b6dbf49706ecbffab22b Mon Sep 17 00:00:00 2001 From: kshitij <82637225+kshitijk4poor@users.noreply.github.com> Date: Sun, 2 Aug 2026 22:07:59 +0530 Subject: [PATCH 05/69] chore: map rkt.2@hotmail.com to sparkeros --- contributors/emails/rkt.2@hotmail.com | 1 + 1 file changed, 1 insertion(+) create mode 100644 contributors/emails/rkt.2@hotmail.com diff --git a/contributors/emails/rkt.2@hotmail.com b/contributors/emails/rkt.2@hotmail.com new file mode 100644 index 000000000000..2819aa968450 --- /dev/null +++ b/contributors/emails/rkt.2@hotmail.com @@ -0,0 +1 @@ +sparkeros From a50bd2ae3fb5528d262529a6eea3fe4477639007 Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Sun, 2 Aug 2026 22:08:46 +0530 Subject: [PATCH 06/69] chore: add contributor email mapping for JeffStone69 --- .../emails/116476090+JeffStone69@users.noreply.github.com | 1 + 1 file changed, 1 insertion(+) create mode 100644 contributors/emails/116476090+JeffStone69@users.noreply.github.com diff --git a/contributors/emails/116476090+JeffStone69@users.noreply.github.com b/contributors/emails/116476090+JeffStone69@users.noreply.github.com new file mode 100644 index 000000000000..73f820cce9bd --- /dev/null +++ b/contributors/emails/116476090+JeffStone69@users.noreply.github.com @@ -0,0 +1 @@ +JeffStone69 From 9929743b729a24e6f25494ff8a7fbdf900396a9a Mon Sep 17 00:00:00 2001 From: JeffStone69 <116476090+JeffStone69@users.noreply.github.com> Date: Fri, 17 Jul 2026 11:09:24 +1000 Subject: [PATCH 07/69] fix(auth): memoize resolve_nous_access_token to collapse startup burst check_tool_availability runs once per managed-tool check_fn (browser, image_gen, etc.) during banner render. Each one independently triggers a ~15s blocking Nous Portal token-refresh network call when the stored token is expired. On a slow/constrained host (e.g. a small monitoring CT) that serial burst stretched startup to many minutes, appearing 'stalled'. Add a per-process memo (5s TTL) so the burst collapses into a single network round-trip. Only successful, non-forced resolutions are cached; force_fresh and insecure/ca_bundle callers bypass and don't populate the cache, so normal refresh semantics are unchanged. Verified: 3 rapid resolve_nous_access_token() calls -> 1 underlying refresh. --- hermes_cli/auth.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/hermes_cli/auth.py b/hermes_cli/auth.py index c08c973045e8..7b8a8fe213e0 100644 --- a/hermes_cli/auth.py +++ b/hermes_cli/auth.py @@ -5741,6 +5741,18 @@ def _agent_key_is_usable(state: Dict[str, Any], min_ttl_seconds: int) -> bool: ) +# Per-process memo for resolve_nous_access_token. Startup runs +# check_tool_availability once per managed-tool check_fn (browser, image_gen, +# etc.), and each one independently triggers a ~15s blocking token-refresh +# network call when the stored token is expired. On a slow/constrained host that +# serial burst stretches startup to many minutes. A short-TTL memo collapses the +# burst into a single network round-trip; callers that need freshness use +# separate flows (force_fresh / refresh_nous_oauth_pure) and are unaffected. +_RESOLVE_TOKEN_CACHE_LOCK = threading.Lock() +_RESOLVE_TOKEN_CACHE: "tuple[float, str] | None" = None +_RESOLVE_TOKEN_CACHE_TTL_S = 5.0 + + def resolve_nous_access_token( *, timeout_seconds: float = 15.0, @@ -5749,6 +5761,16 @@ def resolve_nous_access_token( refresh_skew_seconds: int = ACCESS_TOKEN_REFRESH_SKEW_SECONDS, ) -> str: """Resolve a refresh-aware Nous Portal access token for managed tool gateways.""" + global _RESOLVE_TOKEN_CACHE + # Memo: collapse the startup burst of managed-tool check_fns into one + # network refresh. Only cache a successful, non-forced resolution for a + # short window; force_fresh / error paths bypass and don't populate it. + if not insecure and ca_bundle is None: + with _RESOLVE_TOKEN_CACHE_LOCK: + if _RESOLVE_TOKEN_CACHE is not None: + cached_at, cached_token = _RESOLVE_TOKEN_CACHE + if (time.monotonic() - cached_at) < _RESOLVE_TOKEN_CACHE_TTL_S: + return cached_token with _provider_state_transaction("nous") as ( auth_store, state, @@ -5860,7 +5882,11 @@ def resolve_nous_access_token( } _save_provider_state_to_source(auth_store, "nous", state, state_source_path) _write_shared_nous_state(state) - return state["access_token"] + resolved = state["access_token"] + if not insecure and ca_bundle is None: + with _RESOLVE_TOKEN_CACHE_LOCK: + _RESOLVE_TOKEN_CACHE = (time.monotonic(), resolved) + return resolved def refresh_nous_oauth_pure( From 06b4f64c31d1ba20006c5ef8dea446af9b9b7c45 Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Sun, 2 Aug 2026 22:08:31 +0530 Subject: [PATCH 08/69] fix(auth): memoize the valid-token fast path too, add memo tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-ups on the startup-burst memo: - Populate the memo on the valid-token fast path as well. The startup burst usually finds a VALID token, and each check_fn call still paid two cross-process file locks + state reads to reach that return; the original memo only engaged after a refresh. The token has at least refresh_skew_seconds (>=120s) of life at that return, so a 5s memo can never serve an expired token. - Clear the module-level memo in test_nous_portal_staging_allowlist's refresh-capture helper: with the fast-path populate, a token memoized by an earlier test would otherwise short-circuit the refresh these tests assert on (3 tests failed without this). - Add dedicated memo behavior tests (TTL hit, TTL expiry, insecure bypass) — the original PR shipped none. Mutation-checked: all 3 fail against main's un-memoized function, pass on this branch. --- hermes_cli/auth.py | 9 ++ .../test_nous_portal_staging_allowlist.py | 5 + tests/hermes_cli/test_resolve_token_memo.py | 96 +++++++++++++++++++ 3 files changed, 110 insertions(+) create mode 100644 tests/hermes_cli/test_resolve_token_memo.py diff --git a/hermes_cli/auth.py b/hermes_cli/auth.py index 7b8a8fe213e0..82eb5ee7db5e 100644 --- a/hermes_cli/auth.py +++ b/hermes_cli/auth.py @@ -5825,6 +5825,15 @@ def resolve_nous_access_token( if not _is_expiring(state.get("expires_at"), refresh_skew_seconds): if merged_shared: _save_provider_state_to_source(auth_store, "nous", state, state_source_path) + # Populate the memo on the valid-token fast path too: the + # startup burst usually finds a *valid* token, but each + # check_fn call still pays two cross-process file locks and + # state reads to reach this return. The token has at least + # refresh_skew_seconds (>= 120s) of life here, so a 5s memo + # can never serve an expired token. + if not insecure and ca_bundle is None: + with _RESOLVE_TOKEN_CACHE_LOCK: + _RESOLVE_TOKEN_CACHE = (time.monotonic(), access_token) return access_token if not isinstance(refresh_token, str) or not refresh_token: diff --git a/tests/hermes_cli/test_nous_portal_staging_allowlist.py b/tests/hermes_cli/test_nous_portal_staging_allowlist.py index 71cf4a4981fa..34e02049dd9b 100644 --- a/tests/hermes_cli/test_nous_portal_staging_allowlist.py +++ b/tests/hermes_cli/test_nous_portal_staging_allowlist.py @@ -87,6 +87,11 @@ def _write_auth_file(self, tmp_path, *, stored_portal_url): def _run_and_capture(self, monkeypatch, auth): seen_portal_urls = [] + # The resolve memo is module-level state; clear it so each test's + # resolution actually exercises the refresh path instead of serving + # a token cached by a previous test. + monkeypatch.setattr(auth, "_RESOLVE_TOKEN_CACHE", None) + def _fake_refresh(*, client, portal_base_url, client_id, refresh_token): seen_portal_urls.append(portal_base_url) return { diff --git a/tests/hermes_cli/test_resolve_token_memo.py b/tests/hermes_cli/test_resolve_token_memo.py new file mode 100644 index 000000000000..8fabb8d64c52 --- /dev/null +++ b/tests/hermes_cli/test_resolve_token_memo.py @@ -0,0 +1,96 @@ +"""Tests for the resolve_nous_access_token startup-burst memo (PR #66016). + +The memo collapses the startup burst of managed-tool check_fn calls into a +single expensive resolution: within the short TTL, repeat calls return the +cached token without re-entering _provider_state_transaction (two +cross-process file locks + state reads) or triggering a network refresh. +""" + +import json +import time + +import pytest + +import hermes_cli.auth as auth + + +@pytest.fixture(autouse=True) +def _fresh_memo(monkeypatch, tmp_path): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.delenv("HERMES_PORTAL_BASE_URL", raising=False) + monkeypatch.delenv("NOUS_PORTAL_BASE_URL", raising=False) + monkeypatch.setattr(auth, "_RESOLVE_TOKEN_CACHE", None) + yield + + +def _write_valid_auth_file(tmp_path, token="memo-token"): + (tmp_path / "auth.json").write_text( + json.dumps( + { + "version": 1, + "active_provider": "nous", + "providers": { + "nous": { + "access_token": token, + "refresh_token": "r", + "client_id": "hermes-cli-vps", + "expires_at": time.strftime( + "%Y-%m-%dT%H:%M:%S+00:00", time.gmtime(time.time() + 3600) + ), + } + }, + } + ) + ) + + +def _count_transactions(monkeypatch): + calls = {"n": 0} + real = auth._provider_state_transaction + + def _counting(provider): + calls["n"] += 1 + return real(provider) + + monkeypatch.setattr(auth, "_provider_state_transaction", _counting) + return calls + + +def test_repeat_calls_within_ttl_hit_memo(monkeypatch, tmp_path): + _write_valid_auth_file(tmp_path) + calls = _count_transactions(monkeypatch) + + first = auth.resolve_nous_access_token() + second = auth.resolve_nous_access_token() + third = auth.resolve_nous_access_token() + + assert first == second == third == "memo-token" + assert calls["n"] == 1, ( + "repeat calls within the TTL must not re-enter the state transaction" + ) + + +def test_memo_expires_after_ttl(monkeypatch, tmp_path): + _write_valid_auth_file(tmp_path) + calls = _count_transactions(monkeypatch) + + auth.resolve_nous_access_token() + cached_at, tok = auth._RESOLVE_TOKEN_CACHE + monkeypatch.setattr( + auth, + "_RESOLVE_TOKEN_CACHE", + (cached_at - auth._RESOLVE_TOKEN_CACHE_TTL_S - 1.0, tok), + ) + auth.resolve_nous_access_token() + + assert calls["n"] == 2, "an expired memo must re-resolve" + + +def test_insecure_callers_bypass_memo(monkeypatch, tmp_path): + _write_valid_auth_file(tmp_path) + calls = _count_transactions(monkeypatch) + + auth.resolve_nous_access_token() + auth.resolve_nous_access_token(insecure=True) + + assert calls["n"] == 2, "insecure callers must bypass the memo entirely" From d15b638a88141e2ed6e948f218f7d2027d28f98b Mon Sep 17 00:00:00 2001 From: Shaun Prince Date: Sun, 2 Aug 2026 12:26:52 +0530 Subject: [PATCH 09/69] fix(compression): let explicit interrupts cancel safely Makes interrupt-protected context compression cancellable by an explicit user or lifecycle stop, without weakening protection against ordinary incoming messages, voice interjections, or active-turn redirects. Separates explicit hard cancellation from ordinary interrupt/redirect state with a dedicated threading.Event; introduces AuxiliaryExplicitCancellation as an attempt-local frozen-cause signal; isolates the synchronous provider callback in a bounded daemon worker during protected compression; atomically linearizes Codex timeout cleanup against explicit cancellation; propagates hard cancellation through child agents and explicit stop surfaces; serializes hard-cancel admission against compression commit admission with CompressionCommitFence; aborts before session rotation or late DB commit, restores in-place transcript mutations and compressor state, and releases the heartbeat and compression lease. Based on #74449 by @suparious. Resolved merge conflicts in agent/context_compressor.py (feasibility check + try/except) and tui_gateway/methods_session.py. --- acp_adapter/server.py | 5 +- agent/agent_init.py | 3 + agent/auxiliary_client.py | 257 +++++++- agent/context_compressor.py | 37 +- agent/conversation_compression.py | 344 +++++++++- agent/interrupt_compat.py | 35 + agent/subagent_lifecycle.py | 11 +- cli.py | 16 +- .../emails/suparious@users.noreply.github.com | 1 + cron/scheduler.py | 4 +- gateway/platforms/api_server.py | 9 +- gateway/run.py | 9 +- hermes_state.py | 86 +++ run_agent.py | 99 ++- .../test_auxiliary_explicit_cancellation.py | 622 ++++++++++++++++++ .../agent/test_compression_concurrent_fork.py | 509 +++++++++++++- .../test_compression_interrupt_protection.py | 91 +++ tests/agent/test_interrupt_compat.py | 103 +++ tests/agent/test_subagent_lifecycle.py | 17 + tests/run_agent/test_interrupt_propagation.py | 33 + tests/tui_gateway/test_compute_host.py | 43 ++ tests/tui_gateway/test_protocol.py | 53 +- tools/delegate_tool.py | 14 +- tui_gateway/compute_host.py | 10 +- tui_gateway/methods_session.py | 6 +- 25 files changed, 2338 insertions(+), 79 deletions(-) create mode 100644 agent/interrupt_compat.py create mode 100644 contributors/emails/suparious@users.noreply.github.com create mode 100644 tests/agent/test_auxiliary_explicit_cancellation.py create mode 100644 tests/agent/test_interrupt_compat.py diff --git a/acp_adapter/server.py b/acp_adapter/server.py index f6e0462ec9d1..46577c4fa529 100644 --- a/acp_adapter/server.py +++ b/acp_adapter/server.py @@ -78,6 +78,7 @@ COMPRESSED_SUMMARY_METADATA_KEY, ContextCompressor, ) +from agent.interrupt_compat import request_hard_interrupt from tools.approval import ( reset_hermes_interactive_context, set_hermes_interactive_context, @@ -1547,8 +1548,8 @@ async def cancel(self, session_id: str, **kwargs: Any) -> None: # redirectable work. state.cancel_event.set() try: - if getattr(state, "agent", None) and hasattr(state.agent, "interrupt"): - state.agent.interrupt() + if getattr(state, "agent", None): + request_hard_interrupt(state.agent) except Exception: logger.debug( "Failed to interrupt ACP session %s", diff --git a/agent/agent_init.py b/agent/agent_init.py index d36d1607a897..68b41a231b39 100644 --- a/agent/agent_init.py +++ b/agent/agent_init.py @@ -765,6 +765,9 @@ def init_agent( # Interrupt mechanism for breaking out of tool loops agent._interrupt_requested = False agent._interrupt_message = None # Optional message that triggered interrupt + # Explicit hard cancellation is separate from redirect/message state. A + # thread-safe Event makes the cause atomic for auxiliary stream pollers. + agent._hard_interrupt_requested = threading.Event() agent._execution_thread_id: int | None = None # Set at run_conversation() start agent._interrupt_thread_signal_pending = False agent._client_lock = threading.RLock() diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index 8decf5dd22c4..980813f12358 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -228,30 +228,134 @@ def _create_openai_client(*, api_key: str, base_url: str, **kwargs: Any) -> Any: # part-way, compression falls back to a static "summary unavailable" marker # and the real handoff is lost (#23975). A thread-local flag lets such a # task mark its in-flight LLM call as interrupt-protected; the Codex -# Responses stream's cancellation check honors it. TIMEOUTS still fire +# Responses stream's cancellation check honors it. An explicit host cancel +# (CLI Ctrl+C or /stop) may install a cancel check that overrides protection; +# ordinary incoming-message interrupts remain protected. TIMEOUTS still fire # (a hung call must die), and all OTHER aux tasks (vision, web_extract, # title_generation, …) remain freely interruptible. _aux_interrupt_protection = threading.local() +class AuxiliaryExplicitCancellation(BaseException): + """Frozen signal that an auxiliary attempt was explicitly hard-cancelled. + + This deliberately follows ``asyncio.CancelledError`` and inherits directly + from ``BaseException``: provider retry/fallback code catches ``Exception`` + broadly and must never reinterpret an explicit host stop as a transport + failure. ``cause`` is immutable class data so downstream compression code + does not re-query a mutable host Event after the transport has unwound. + """ + + cause = "explicit_host_cancel" + + def __init__(self) -> None: + super().__init__("auxiliary request explicitly cancelled by host") + + def _aux_interrupt_protected() -> bool: return bool(getattr(_aux_interrupt_protection, "active", False)) +def _aux_interrupt_cancel_requested() -> bool: + """Return whether an explicit host cancel overrides aux protection.""" + event = getattr(_aux_interrupt_protection, "cancel_event", None) + if event is not None: + try: + return bool(event.is_set()) + except Exception: + logger.debug("aux interrupt cancel event check failed", exc_info=True) + return False + check = getattr(_aux_interrupt_protection, "cancel_check", None) + if not callable(check): + return False + try: + return bool(check()) + except Exception: + logger.debug("aux interrupt cancel check failed", exc_info=True) + return False + + @contextlib.contextmanager -def aux_interrupt_protection(active: bool = True): +def aux_interrupt_protection( + active: bool = True, + cancel_check=None, + cancel_event=None, +): """Mark the current thread's auxiliary LLM call as interrupt-protected. Used by atomic aux tasks (compression) so a mid-flight gateway interrupt doesn't abort the call and trigger a degraded fallback. Re-entrant-safe: - restores the previous value on exit. + restores the previous value on exit. ``cancel_check`` lets the host retain + an explicit hard-cancel path; ``cancel_event`` is preferred when the host + already owns an Event. Nested protection scopes inherit both values. """ prev = getattr(_aux_interrupt_protection, "active", False) + prev_cancel_check = getattr(_aux_interrupt_protection, "cancel_check", None) + prev_cancel_event = getattr(_aux_interrupt_protection, "cancel_event", None) _aux_interrupt_protection.active = active + if callable(cancel_check): + _aux_interrupt_protection.cancel_check = cancel_check + if cancel_event is not None and callable(getattr(cancel_event, "is_set", None)): + _aux_interrupt_protection.cancel_event = cancel_event try: yield finally: _aux_interrupt_protection.active = prev + _aux_interrupt_protection.cancel_check = prev_cancel_check + _aux_interrupt_protection.cancel_event = prev_cancel_event + + +def _capture_aux_cancel_check() -> Optional[Callable[[], Any]]: + """Capture the current explicit-cancel source on the owning request thread.""" + event = getattr(_aux_interrupt_protection, "cancel_event", None) + is_set = getattr(event, "is_set", None) + if callable(is_set): + return is_set + check = getattr(_aux_interrupt_protection, "cancel_check", None) + if callable(check): + # Preserve callable identity so attempt-local decision objects retain + # methods such as begin_timeout_cleanup() when captured by adapters. + return check + return None + + +def _captured_aux_cancel_requested(cancel_check: Callable[[], Any]) -> bool: + """Read a request-thread cancellation source without leaking its failures.""" + try: + return bool(cancel_check()) + except Exception: + logger.debug("captured aux cancel check failed", exc_info=True) + return False + + +class _AuxiliaryCancellationDecision: + """Atomically choose explicit cancellation or provider timeout per attempt.""" + + def __init__(self, source_cancel_check: Callable[[], Any]) -> None: + self._source_cancel_check = source_cancel_check + self._lock = threading.Lock() + self._outcome = "active" + + def __call__(self) -> bool: + with self._lock: + if self._outcome == "cancelled": + return True + if self._outcome == "timed_out": + return False + if _captured_aux_cancel_requested(self._source_cancel_check): + self._outcome = "cancelled" + return True + return False + + def begin_timeout_cleanup(self) -> bool: + """Return whether timeout won and destructive cleanup is permitted.""" + with self._lock: + if self._outcome == "active": + if _captured_aux_cancel_requested(self._source_cancel_check): + self._outcome = "cancelled" + else: + self._outcome = "timed_out" + return self._outcome == "timed_out" # ── Forward-progress hook for streamed auxiliary calls ─────────────────── @@ -298,6 +402,75 @@ def aux_progress_hook(hook): _aux_progress.hook = prev +def _run_protected_sync_provider_call( + callback: Callable[[dict[str, Any]], Any], + kwargs: dict[str, Any], +) -> Any: + """Run one protected provider callback in an attempt-isolated daemon. + + A hard cancel must release the compression-owning thread promptly, but + auxiliary clients are process-shared and cannot safely be closed or evicted + to wake one request. Only protected calls with a captured hard-cancel source + use this seam. Their provider callback (including stream aggregation) runs + in a daemon worker while the owner polls cancellation. On cancel the owner + unwinds immediately; the worker is left to finish under the provider timeout + already present in ``kwargs``. It owns no transcript or compressor commit + state and never holds the session lock. + + Ordinary auxiliary calls, and protected calls without a cancellation source, + retain the historical direct synchronous path with no extra thread. + """ + source_cancel_check = _capture_aux_cancel_check() + if not _aux_interrupt_protected() or not callable(source_cancel_check): + return callback(kwargs) + + # Freeze one linearized outcome for this isolated attempt. The host Event is + # reused and cleared on a later turn, while the Codex timeout Timer may race + # owner polling. Both paths must decide under the same attempt-local lock. + cancel_check = _AuxiliaryCancellationDecision(source_cancel_check) + + if cancel_check(): + raise AuxiliaryExplicitCancellation() + + progress_hook = getattr(_aux_progress, "hook", None) + provider_context = contextvars.copy_context() + done = threading.Event() + outcome: dict[str, Any] = {} + + def _provider_worker() -> None: + try: + with aux_progress_hook(progress_hook), aux_interrupt_protection( + cancel_check=cancel_check + ): + outcome["result"] = callback(kwargs) + except BaseException as exc: + outcome["exception"] = exc + finally: + done.set() + + threading.Thread( + target=provider_context.run, + args=(_provider_worker,), + name="hermes-protected-aux-provider", + daemon=True, + ).start() + + while True: + # Cancellation is checked before and after every completion wait so it + # wins whenever result publication and the host Event become visible in + # the same polling interval. + if _captured_aux_cancel_requested(cancel_check): + raise AuxiliaryExplicitCancellation() + if not done.wait(0.02): + continue + if _captured_aux_cancel_requested(cancel_check): + raise AuxiliaryExplicitCancellation() + exception = outcome.get("exception") + if exception is not None: + raise exception + return outcome.get("result") + + def _safe_isinstance(obj: Any, maybe_type: Any) -> bool: """Return False instead of raising when a patched symbol is not a type.""" try: @@ -1155,12 +1328,53 @@ def create(self, **kwargs) -> Any: deadline = time.monotonic() + float(total_timeout) if total_timeout else None timed_out = threading.Event() timeout_timer: Optional[threading.Timer] = None + # A protected provider call may outlive its owning compression attempt: + # the owner returns promptly on hard cancellation while this adapter is + # still blocked in the SDK stream on its isolated worker. Timer threads + # do not inherit this worker's thread-local protection state, so freeze + # the hard-cancel source here, before creating the timer. + protected_cancel_check = ( + _capture_aux_cancel_check() if _aux_interrupt_protected() else None + ) + attempt_stream_lock = threading.Lock() + attempt_stream: List[Any] = [] def _timeout_message() -> str: return f"Codex auxiliary Responses stream exceeded {float(total_timeout):.1f}s total timeout" def _close_client_on_timeout() -> None: + begin_timeout_cleanup = getattr( + protected_cancel_check, "begin_timeout_cleanup", None + ) + if callable(begin_timeout_cleanup): + timeout_won = bool(begin_timeout_cleanup()) + else: + timeout_won = not ( + callable(protected_cancel_check) + and _captured_aux_cancel_requested(protected_cancel_check) + ) + # Publish transport timeout only after the attempt-local decision is + # fixed, so owner polling cannot observe completion in between. timed_out.set() + if not timeout_won: + # The request owner already hard-cancelled this attempt. The + # OpenAI client is process-shared, so closing/evicting it here + # would disrupt unrelated sessions. Wake only this attempt's + # event stream when responses.create() returned one in time; + # otherwise rely on the bounded SDK/provider timeout. + with attempt_stream_lock: + stream = attempt_stream[0] if attempt_stream else None + close_stream = getattr(stream, "close", None) + if callable(close_stream): + try: + close_stream() + except Exception: + logger.debug( + "Codex auxiliary: cancelled attempt stream close " + "during timeout failed", + exc_info=True, + ) + return close = getattr(self._client, "close", None) if callable(close): try: @@ -1187,11 +1401,14 @@ def _check_cancelled() -> None: from tools.interrupt import is_interrupted # Honor interrupt protection for atomic aux tasks (compression): # a mid-flight gateway interrupt must NOT abort the summary call - # and trigger a degraded fallback marker (#23975). Timeouts above - # still fire; other aux tasks remain interruptible. + # and trigger a degraded fallback marker (#23975). Explicit host + # cancellation has its own frozen exception; timeouts above still + # fire and other aux tasks remain interruptible. + if _aux_interrupt_cancel_requested(): + raise AuxiliaryExplicitCancellation() if is_interrupted() and not _aux_interrupt_protected(): raise InterruptedError("Codex auxiliary Responses stream interrupted") - except InterruptedError: + except (InterruptedError, AuxiliaryExplicitCancellation): raise except Exception: # Interrupt state is a best-effort UX hook; never make it a @@ -1230,6 +1447,25 @@ def _on_each_event(_event: Any) -> None: _check_cancelled() event_stream = self._client.responses.create(**stream_kwargs) + with attempt_stream_lock: + attempt_stream.append(event_stream) + # The timer can fire while responses.create() is blocked. If the + # cancelled attempt had no stream to close at that instant, close it + # now that it is safely attempt-owned; never touch the shared client. + if ( + timed_out.is_set() + and callable(protected_cancel_check) + and _captured_aux_cancel_requested(protected_cancel_check) + ): + close_fn = getattr(event_stream, "close", None) + if callable(close_fn): + try: + close_fn() + except Exception: + logger.debug( + "Codex auxiliary: late cancelled attempt stream close failed", + exc_info=True, + ) try: # Some Codex-compatible hosts accept ``stream=True`` but return # a completed Responses object instead of an SSE iterator. Do @@ -1251,6 +1487,8 @@ def _on_each_event(_event: Any) -> None: close_fn() except Exception: pass + with attempt_stream_lock: + attempt_stream.clear() if final is None: raise RuntimeError("Codex auxiliary Responses stream did not return a final response") @@ -2672,14 +2910,17 @@ def _relay_sync_completion( ) -> Any: callback = create or (lambda request: client.chat.completions.create(**request)) route = _relay_auxiliary_metadata(provider=provider, api_mode=api_mode) + # Protected compression calls isolate only the provider callback and stream + # aggregation. The owning thread remains free to unwind its lease/DB + # transaction on hard cancel without touching the process-shared client. if route is None: - return callback(kwargs) + return _run_protected_sync_provider_call(callback, kwargs) provider_name, fallback_model, metadata = route from agent import relay_llm return relay_llm.execute_current( kwargs, - callback, + lambda request: _run_protected_sync_provider_call(callback, request), name=provider_name, model_name=str(kwargs.get("model") or fallback_model), metadata=metadata, diff --git a/agent/context_compressor.py b/agent/context_compressor.py index e8688d253b2f..9e93ac23d444 100644 --- a/agent/context_compressor.py +++ b/agent/context_compressor.py @@ -25,7 +25,12 @@ import uuid from typing import Any, Dict, List, Optional -from agent.auxiliary_client import call_llm, _is_connection_error, aux_interrupt_protection +from agent.auxiliary_client import ( + AuxiliaryExplicitCancellation, + _is_connection_error, + aux_interrupt_protection, + call_llm, +) from agent.context_engine import ContextEngine, sanitize_memory_context from agent.error_classifier import FailoverReason, classify_api_error from agent.model_metadata import ( @@ -1828,6 +1833,11 @@ def get_active_compression_failure_cooldown( refresh: bool = False, ) -> Optional[Dict[str, Any]]: """Return the live compression-failure cooldown for the bound session.""" + if refresh: + # Transaction rollback must distinguish an authoritative empty row + # from a failed/unavailable durable read. The public return value + # cannot do so because it deliberately falls back to local state. + self._last_cooldown_refresh_was_authoritative = None now_mono = time.monotonic() local_state = None if self._summary_failure_cooldown_until > now_mono: @@ -1852,10 +1862,16 @@ def get_active_compression_failure_cooldown( try: state = getter(session_id) except sqlite3.Error as exc: + if refresh: + self._last_cooldown_refresh_was_authoritative = False logger.debug("compression failure cooldown lookup failed: %s", exc) return local_state except Exception: + if refresh: + self._last_cooldown_refresh_was_authoritative = False return local_state + if refresh: + self._last_cooldown_refresh_was_authoritative = True if not state: if refresh: if local_state is not None and self._cooldown_persist_failed: @@ -6056,6 +6072,7 @@ def compress( # — take the narrow rescan, miss a beyond-window fossil, and discard the # rehydrated state as cross-session leakage (#57835). _previous_summary_before_scan = self._previous_summary + _summary_has_user_turn_before_scan = getattr(self, "_summary_has_user_turn", None) # A persisted handoff summary can sit in the protected head after a # resume (commonly immediately after the system prompt). Search from # the first non-system message through the compression window. On the @@ -6264,11 +6281,19 @@ def _window_row(idx: int, msg: Dict[str, Any]): # Deriving the auto focus topic scans recent user turns — only pay # for it when a summary will actually be generated. summary_focus_topic = focus_topic or self._derive_auto_focus_topic(messages) - summary = self._generate_summary( - turns_to_summarize, - focus_topic=summary_focus_topic, - memory_context=memory_context, - ) + try: + summary = self._generate_summary( + turns_to_summarize, + focus_topic=summary_focus_topic, + memory_context=memory_context, + ) + except AuxiliaryExplicitCancellation: + # Explicit cancellation is a true no-op. Restore state mutated by + # the resume/handoff self-heal scan before the exception escapes to + # the outer transaction, which restores the transcript and lease. + self._previous_summary = _previous_summary_before_scan + self._summary_has_user_turn = _summary_has_user_turn_before_scan + raise # If summary generation failed, behavior splits on # ``abort_on_summary_failure`` (config: compression.abort_on_summary_failure): diff --git a/agent/conversation_compression.py b/agent/conversation_compression.py index 81b8ca71ac16..93a295e27cd8 100644 --- a/agent/conversation_compression.py +++ b/agent/conversation_compression.py @@ -42,6 +42,7 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Tuple +from agent.auxiliary_client import AuxiliaryExplicitCancellation from agent.context_engine import ( automatic_compaction_status_message, sanitize_memory_context, @@ -207,6 +208,202 @@ def _cached_prompt_reflects_builtin_memory(agent: Any, cached_prompt: str) -> bo return True +_COMPRESSOR_ATTEMPT_STATE_FIELDS = ( + "_previous_summary", + "_summary_has_user_turn", + "compression_count", + "_last_compression_savings_pct", + "_ineffective_compression_count", + "_anti_thrash_recovery_deadline", + "_fallback_compression_streak", + "_verify_compaction_cleared_threshold", + "_last_compression_made_progress", + "_summary_failure_cooldown_until", + "_cooldown_persist_failed", + "_last_summary_error", + "_consecutive_timeout_failures", + "_last_summary_dropped_count", + "_last_summary_fallback_used", + "_last_compress_aborted", + "_last_summary_auth_failure", + "_last_summary_network_failure", + "_last_aux_model_failure_error", + "_last_aux_model_failure_model", + "_summary_model_fallen_back", + "summary_model", + "_last_compression_telemetry", + "_active_compression_telemetry", + "_compression_telemetry_seed", +) + +_COMPRESSOR_COOLDOWN_STATE_FIELDS = ( + "_summary_failure_cooldown_until", + "_last_summary_error", + "_cooldown_persist_failed", +) + + +def _snapshot_compressor_attempt_state(compressor: Any) -> dict[str, Any]: + """Copy only mutable bookkeeping owned by one compression attempt. + + The explicit allow-list avoids copying provider clients, SessionDB handles, + locks, and plugin resources. Missing fields are intentionally ignored so + legacy and third-party compressors keep their existing contract. + """ + try: + values = vars(compressor) + except TypeError: + return {} + selected = { + name: values[name] + for name in _COMPRESSOR_ATTEMPT_STATE_FIELDS + if name in values + } + # Copy the collection as one object so aliases between fields (notably + # _active_compression_telemetry and _last_compression_telemetry) survive. + return copy.deepcopy(selected) + + +def _restore_compressor_attempt_state( + compressor: Any, + snapshot: dict[str, Any], + *, + durable_cooldown_authoritative: Optional[bool] = None, + durable_cooldown_state: Optional[dict[str, Any]] = None, +) -> None: + """Restore the safe per-attempt snapshot after a pre-commit hard cancel.""" + # A successful summary clears the durable cooldown before the outer commit + # boundary. Recreate (or clear) that row before restoring exact in-memory + # values, otherwise the next refresh would overwrite this rollback. Unknown + # durable state and intentionally unpersisted local cooldowns are never + # converted into destructive DB writes during cancellation. + if ( + "_summary_failure_cooldown_until" in snapshot + and durable_cooldown_authoritative is not False + and ( + durable_cooldown_authoritative is True + or not bool(snapshot.get("_cooldown_persist_failed", False)) + ) + ): + session_db = vars(compressor).get("_session_db") + session_id = vars(compressor).get("_session_id") + if session_db is not None and session_id: + if durable_cooldown_authoritative is True: + restorer = getattr( + type(session_db), + "restore_compression_failure_cooldown_row", + None, + ) + if not callable(restorer) or durable_cooldown_state is None: + raise RuntimeError( + "exact compression cooldown rollback API is unavailable" + ) + # This API restores raw columns (including expired and null + # combinations), verifies the read-back, and propagates failure. + restorer( + session_db, + session_id, + copy.deepcopy(durable_cooldown_state), + ) + else: + try: + deadline = float( + snapshot["_summary_failure_cooldown_until"] or 0.0 + ) + remaining = max(0.0, deadline - time.monotonic()) + durable_deadline = time.time() + remaining + durable_error = snapshot.get("_last_summary_error") + if remaining > 0: + recorder = getattr( + type(session_db), + "record_compression_failure_cooldown", + None, + ) + if callable(recorder): + recorder( + session_db, + session_id, + durable_deadline, + durable_error, + ) + else: + clearer = getattr( + type(session_db), + "clear_compression_failure_cooldown", + None, + ) + if callable(clearer): + clearer(session_db, session_id) + except Exception: + # Legacy/third-party compatibility path: its existing APIs + # do not provide a verifiable transaction contract. + logger.debug( + "compression cooldown persistence rollback failed", + exc_info=True, + ) + restored = copy.deepcopy(snapshot) + for name, value in restored.items(): + setattr(compressor, name, value) + + +def _capture_authoritative_cooldown_under_lease( + compressor: Any, + attempt_snapshot: dict[str, Any], +) -> tuple[Optional[bool], Optional[dict[str, Any]]]: + """Refresh and snapshot built-in durable cooldown state under the lease. + + Third-party compressors are deliberately not invoked here: arbitrary plugin + callbacks must not run while the session lease is held. A durable read + failure returns ``False`` so rollback cannot mistake unknown durable state + for an authoritative empty row and clear it; an unavailable legacy API + returns ``None`` and preserves the compatibility path. + """ + try: + from agent.context_compressor import ContextCompressor + + if not isinstance(compressor, ContextCompressor): + return None, None + values = vars(compressor) + session_db = values.get("_session_db") + session_id = values.get("_session_id") + raw_reader = ( + getattr( + type(session_db), "get_compression_failure_cooldown_row", None + ) + if session_db is not None + else None + ) + if session_db is None or not session_id: + # Unbound compressors have no durable row to mutate or restore. + return None, None + if not callable(raw_reader): + return False, None + # Capture the exact persisted representation first. The active getter + # intentionally filters expired rows and therefore cannot serve as a + # lossless rollback snapshot. + durable_state = raw_reader(session_db, session_id) + if not isinstance(durable_state, dict): + raise TypeError("raw compression cooldown snapshot must be a mapping") + ContextCompressor.get_active_compression_failure_cooldown( + compressor, + refresh=True, + ) + except Exception as exc: + logger.debug("authoritative compression cooldown capture failed: %s", exc) + return False, None + authoritative = getattr( + compressor, "_last_cooldown_refresh_was_authoritative", None + ) + if authoritative is not True: + return authoritative, None + + values = vars(compressor) + for name in _COMPRESSOR_COOLDOWN_STATE_FIELDS: + if name in values: + attempt_snapshot[name] = copy.deepcopy(values[name]) + return True, copy.deepcopy(durable_state) + + class CompressionCommitFence: """Fence timeout cancellation against post-summary session mutation. @@ -241,7 +438,7 @@ def seconds_since_progress(self) -> float: """Seconds since the worker last reported forward progress.""" return max(0.0, time.monotonic() - self._last_progress) - def cancel_before_commit(self) -> bool: + def cancel_before_commit(self, cancel_event: Any = None) -> bool: """Cancel a pending commit, or wait for an active commit to finish. Returns ``True`` when cancellation won before the commit boundary. @@ -250,8 +447,12 @@ def cancel_before_commit(self) -> bool: """ with self._lock: if self._commit_started: + if cancel_event is not None: + cancel_event.set() return False self._cancelled = True + if cancel_event is not None: + cancel_event.set() return True def try_cancel_before_commit(self) -> Optional[bool]: @@ -270,10 +471,13 @@ def try_cancel_before_commit(self) -> Optional[bool]: finally: self._lock.release() - def begin_commit(self) -> bool: - """Enter the commit boundary unless cancellation already won.""" + def begin_commit(self, cancel_event: Any = None) -> bool: + """Atomically admit commit unless a hard cancellation already won.""" self._lock.acquire() - if self._cancelled: + if self._cancelled or ( + cancel_event is not None and bool(cancel_event.is_set()) + ): + self._cancelled = True self._lock.release() return False self._commit_started = True @@ -307,13 +511,21 @@ def _lock_api_is_absent_on_session_db(lock_db: Any) -> bool: return False -def _refresh_persisted_compression_guards(compressor: Any) -> None: +def _refresh_persisted_compression_guards( + compressor: Any, + *, + include_cooldown: bool = True, +) -> None: """Refresh durable automatic-compression guards on a built-in compressor.""" - method_calls = ( - ("get_active_compression_failure_cooldown", {"refresh": True}), + method_calls = [ ("_load_fallback_compression_streak", {}), ("_load_ineffective_compression_count", {}), - ) + ] + if include_cooldown: + method_calls.insert( + 0, + ("get_active_compression_failure_cooldown", {"refresh": True}), + ) for method_name, kwargs in method_calls: method = getattr(type(compressor), method_name, None) if not callable(method): @@ -1298,6 +1510,11 @@ def compress_context( prompt — the session is NOT rotated. Callers should detect the no-op via ``len(returned) == len(input)`` and stop the retry loop. """ + _compressor_attempt_snapshot = _snapshot_compressor_attempt_state( + agent.context_compressor + ) + _durable_cooldown_authoritative: Optional[bool] = None + _durable_cooldown_state: Optional[dict[str, Any]] = None if ( defer_context_engine_notification and callable(getattr(agent, _PENDING_CONTEXT_ENGINE_NOTIFICATION, None)) @@ -1343,8 +1560,13 @@ def compress_context( if getattr(agent, "api_mode", None) == "codex_app_server": _codex_fence_entered = False if commit_fence is not None: - _codex_fence_entered = commit_fence.begin_commit() + _codex_fence_entered = commit_fence.begin_commit( + getattr(agent, "_hard_interrupt_requested", None) + ) if not _codex_fence_entered: + _restore_compressor_attempt_state( + agent.context_compressor, _compressor_attempt_snapshot + ) existing_prompt = getattr(agent, "_cached_system_prompt", None) if not existing_prompt: existing_prompt = agent._build_system_prompt(system_message) @@ -1671,13 +1893,36 @@ def _release_lock() -> None: ) return messages, _existing_sp + # Snapshot the authoritative durable cooldown only after this attempt owns + # the session lease. This runs for force=True too, but does not apply the + # automatic breaker gate: manual compression still retries immediately. + _durable_cooldown_authoritative, _durable_cooldown_state = ( + _capture_authoritative_cooldown_under_lease( + agent.context_compressor, + _compressor_attempt_snapshot, + ) + ) + if _durable_cooldown_authoritative is False: + # A bound built-in compressor reached its durable getter and the read + # failed. Proceeding with force=True could clear an unknown newer row + # before cancellation has enough information to restore it. This is a + # persistence-safety abort, not automatic breaker gating. + _release_lock() + existing_prompt = getattr(agent, "_cached_system_prompt", None) + if not existing_prompt: + existing_prompt = agent._build_system_prompt(system_message) + return messages, existing_prompt + # The agent may have been constructed before another path completed an # in-place compaction on the same session. Re-read durable breaker state # after acquiring the session lock so this final gate cannot act on the # stale snapshot loaded by bind_session_state(). if not force: compressor = agent.context_compressor - _refresh_persisted_compression_guards(compressor) + _refresh_persisted_compression_guards( + compressor, + include_cooldown=False, + ) blocked = getattr( type(compressor), "_automatic_compression_blocked", @@ -1691,6 +1936,7 @@ def _release_lock() -> None: return messages, existing_prompt _activity_heartbeat: Optional[_CompressionActivityHeartbeat] = None + messages_before_compression = None try: if _lock_holder is not None: _lock_refresher = _CompressionLockLeaseRefresher( @@ -1799,13 +2045,74 @@ def _release_lock() -> None: # provider that keeps the connection alive forever is cut off at the # streamed total ceiling (see _aux_stream_total_ceiling) instead of # outliving the SDK's inactivity timeout indefinitely. - from agent.auxiliary_client import aux_progress_hook + from agent.auxiliary_client import ( + aux_interrupt_protection, + aux_progress_hook, + ) _progress_hook = ( commit_fence.touch_progress if commit_fence is not None else (lambda: None) ) - with aux_progress_hook(_progress_hook): + # Incoming-message interrupts and active-turn redirects must not tear an + # atomic summary in half (#23975). Explicit stop surfaces set a separate + # Event atomically; never infer cause from the racy message fields. + _hard_cancel_event = getattr(agent, "_hard_interrupt_requested", None) + with aux_progress_hook(_progress_hook), aux_interrupt_protection( + cancel_event=_hard_cancel_event + ): compressed = compress_fn(messages, **compress_kwargs) + # Freeze a hard stop that arrived after the final provider attempt + # unwound but before this transaction can rotate session state. + if _hard_cancel_event is not None and _hard_cancel_event.is_set(): + raise AuxiliaryExplicitCancellation() + except AuxiliaryExplicitCancellation: + try: + _restore_compressor_attempt_state( + agent.context_compressor, + _compressor_attempt_snapshot, + durable_cooldown_authoritative=_durable_cooldown_authoritative, + durable_cooldown_state=_durable_cooldown_state, + ) + except BaseException as _rollback_exc: + # Compensation failure must surface, but it must not strand the + # session lease or retain an in-memory transcript mutation. + if ( + messages_before_compression is not None + and messages != messages_before_compression + ): + messages[:] = copy.deepcopy(messages_before_compression) + if _activity_heartbeat is not None: + _activity_heartbeat.stop("context compression rollback failed") + _activity_heartbeat = None + _release_lock() + _emit_compression_attempt_telemetry( + agent, + started_at=_attempt_started_at, + commit_status="aborted", + split_status="aborted", + failure_class=f"rollback:{type(_rollback_exc).__name__}", + ) + raise + if ( + messages_before_compression is not None + and messages != messages_before_compression + ): + messages[:] = copy.deepcopy(messages_before_compression) + if _activity_heartbeat is not None: + _activity_heartbeat.stop("context compression cancelled") + _activity_heartbeat = None + _release_lock() + _emit_compression_attempt_telemetry( + agent, + started_at=_attempt_started_at, + commit_status="aborted", + split_status="aborted", + failure_class="explicit_interrupt", + ) + _existing_sp = getattr(agent, "_cached_system_prompt", None) + if not _existing_sp: + _existing_sp = agent._build_system_prompt(system_message) + return messages, _existing_sp except BaseException as _compress_exc: # ANY exception after lock acquisition — memory hook, capability # inspection, engine lookup, or compress() — must release the lock so @@ -1918,8 +2225,19 @@ def _release_lock() -> None: return messages, _existing_sp if commit_fence is not None: - _commit_fence_entered = commit_fence.begin_commit() + _commit_fence_entered = commit_fence.begin_commit(_hard_cancel_event) if not _commit_fence_entered: + _restore_compressor_attempt_state( + agent.context_compressor, + _compressor_attempt_snapshot, + durable_cooldown_authoritative=_durable_cooldown_authoritative, + durable_cooldown_state=_durable_cooldown_state, + ) + if ( + messages_before_compression is not None + and messages != messages_before_compression + ): + messages[:] = copy.deepcopy(messages_before_compression) logger.info( "Compression commit cancelled before session mutation " "(session=%s).", diff --git a/agent/interrupt_compat.py b/agent/interrupt_compat.py new file mode 100644 index 000000000000..bf56849495c7 --- /dev/null +++ b/agent/interrupt_compat.py @@ -0,0 +1,35 @@ +"""Compatibility helper for explicit agent stop producers.""" + +from __future__ import annotations + +import inspect +from typing import Any + + +def request_hard_interrupt(agent: Any, message: str | None = None) -> bool: + """Request an explicit stop, falling back to the legacy interrupt ABI. + + New agents expose ``hard_interrupt(message=None)``. Third-party agents and + old test doubles may only expose ``interrupt(message=None)``; keep those + usable without sending the newer ``hard_cancel=`` keyword they do not know. + Returns ``False`` only when neither callable is available. + """ + # Avoid treating a dynamic ``__getattr__`` proxy (notably an unspecced + # ``MagicMock`` or a third-party RPC facade) as if it genuinely implements + # the new ABI. Static lookup proves the attribute exists on the instance or + # its type before normal descriptor binding retrieves the callable. + try: + inspect.getattr_static(agent, "hard_interrupt") + except AttributeError: + interrupt = None + else: + interrupt = getattr(agent, "hard_interrupt", None) + if not callable(interrupt): + interrupt = getattr(agent, "interrupt", None) + if not callable(interrupt): + return False + if message is None: + interrupt() + else: + interrupt(message) + return True diff --git a/agent/subagent_lifecycle.py b/agent/subagent_lifecycle.py index bbe63ad0930a..55e110aae5a6 100644 --- a/agent/subagent_lifecycle.py +++ b/agent/subagent_lifecycle.py @@ -21,6 +21,7 @@ from concurrent.futures import Future, TimeoutError from typing import Any, Callable, Mapping, Optional +from agent.interrupt_compat import request_hard_interrupt PUBLIC_CONTRACT_VERSION = 1 _MAX_GOAL_CHARS = 16_000 @@ -300,16 +301,22 @@ def cancel(self, handle: SubagentHandle, *, reason: str) -> SubagentCancelResult agent = record.agent record.state = SubagentState.CANCEL_REQUESTED record.updated_at = time.time() - if agent is None or not hasattr(agent, "interrupt"): + if agent is None: return SubagentCancelResult( False, unsupported=True, state=SubagentState.CANCEL_REQUESTED ) try: - agent.interrupt(f"Lifecycle cancellation requested: {reason[:500]}") + accepted = request_hard_interrupt( + agent, f"Lifecycle cancellation requested: {reason[:500]}" + ) except Exception: return SubagentCancelResult( False, unsupported=True, state=SubagentState.CANCEL_REQUESTED ) + if not accepted: + return SubagentCancelResult( + False, unsupported=True, state=SubagentState.CANCEL_REQUESTED + ) return SubagentCancelResult(True, state=SubagentState.CANCEL_REQUESTED) def result(self, handle: SubagentHandle) -> SubagentResult: diff --git a/cli.py b/cli.py index 559094a527d8..f62fdc603e95 100644 --- a/cli.py +++ b/cli.py @@ -54,6 +54,7 @@ from hermes_cli.cli_agent_setup_mixin import CLIAgentSetupMixin from hermes_cli.cli_commands_mixin import CLICommandsMixin from hermes_cli.cli_billing_mixin import CLIBillingMixin +from agent.interrupt_compat import request_hard_interrupt # prompt_toolkit for fixed input area TUI from prompt_toolkit.history import FileHistory @@ -15834,7 +15835,7 @@ def handle_ctrl_c(event): self._last_ctrl_c_time = now print("\n⚡ Interrupting agent... (press Ctrl+C again to force exit)") - self.agent.interrupt() + request_hard_interrupt(self.agent) # If there's text or images, clear them (like bash). # If everything is already empty, exit. elif event.app.current_buffer.text or self._attached_images: @@ -15912,7 +15913,7 @@ def handle_ctrl_q(event): if self._agent_running and self.agent: print("\n⚡ Interrupting agent...") - self.agent.interrupt() + request_hard_interrupt(self.agent) elif event.app.current_buffer.text or self._attached_images: event.app.current_buffer.reset() self._attached_images.clear() @@ -17492,8 +17493,11 @@ def _signal_handler(signum, frame): # minutes (#65998 class). Never raises. _arm_exit_watchdog_on_shutdown_signal() try: - if getattr(self, "agent", None) and getattr(self, "_agent_running", False): - self.agent.interrupt(f"received signal {signum}") + _signal_agent = getattr(self, "agent", None) + if _signal_agent is not None and getattr(self, "_agent_running", False): + request_hard_interrupt( + _signal_agent, f"received signal {signum}" + ) try: _grace = float(os.getenv("HERMES_SIGTERM_GRACE", "1.5")) except (TypeError, ValueError): @@ -17684,7 +17688,7 @@ def new_event_loop(self): # avoids wasted API calls and lets run_conversation clean up). if self.agent and getattr(self, '_agent_running', False): try: - self.agent.interrupt() + request_hard_interrupt(self.agent) except Exception: pass # Shut down voice recorder (release persistent audio stream) @@ -18105,7 +18109,7 @@ def _signal_handler_q(signum, frame): try: _agent = getattr(cli, "agent", None) if _agent is not None: - _agent.interrupt(f"received signal {signum}") + request_hard_interrupt(_agent, f"received signal {signum}") try: _grace = float(os.getenv("HERMES_SIGTERM_GRACE", "1.5")) except (TypeError, ValueError): diff --git a/contributors/emails/suparious@users.noreply.github.com b/contributors/emails/suparious@users.noreply.github.com new file mode 100644 index 000000000000..994fe5b8902b --- /dev/null +++ b/contributors/emails/suparious@users.noreply.github.com @@ -0,0 +1 @@ +suparious diff --git a/cron/scheduler.py b/cron/scheduler.py index 327a048becc6..8cae9b69ca70 100644 --- a/cron/scheduler.py +++ b/cron/scheduler.py @@ -48,6 +48,7 @@ ) from hermes_cli.fallback_config import get_fallback_chain from hermes_time import now as _hermes_now +from agent.interrupt_compat import request_hard_interrupt logger = logging.getLogger(__name__) @@ -3632,8 +3633,7 @@ def _heartbeat_run_claim_if_due(): _last_desc, _iter_n, _iter_max, _cur_tool or "none", ) - if hasattr(agent, "interrupt"): - agent.interrupt("Cron job timed out (inactivity)") + request_hard_interrupt(agent, "Cron job timed out (inactivity)") raise TimeoutError( f"Cron job '{job_name}' idle for " f"{int(_secs_ago)}s (limit {int(_cron_inactivity_limit)}s) " diff --git a/gateway/platforms/api_server.py b/gateway/platforms/api_server.py index f51d1d5834c8..8f346a4599e4 100644 --- a/gateway/platforms/api_server.py +++ b/gateway/platforms/api_server.py @@ -92,6 +92,7 @@ def _approval_event_choices(*, smart_denied: bool, allow_permanent: bool) -> lis validate_media_delivery_path, ) from agent.redact import redact_sensitive_text +from agent.interrupt_compat import request_hard_interrupt from gateway.readiness import collect_runtime_readiness logger = logging.getLogger(__name__) @@ -4335,7 +4336,7 @@ async def _emit(item): agent = agent_ref[0] if agent_ref else None if agent is not None: try: - agent.interrupt("SSE client disconnected") + request_hard_interrupt(agent, "SSE client disconnected") except Exception: pass _reap_disconnected_agent_processes(agent) @@ -4915,7 +4916,7 @@ async def _flush_batch() -> None: agent = agent_ref[0] if agent_ref else None if agent is not None: try: - agent.interrupt("SSE client disconnected") + request_hard_interrupt(agent, "SSE client disconnected") except Exception: pass _reap_disconnected_agent_processes(agent) @@ -4935,7 +4936,7 @@ async def _flush_batch() -> None: agent = agent_ref[0] if agent_ref else None if agent is not None: try: - agent.interrupt("SSE task cancelled") + request_hard_interrupt(agent, "SSE task cancelled") except Exception: pass # Same abandonment as a client disconnect: the run will never @@ -6788,7 +6789,7 @@ async def _handle_stop_run(self, request: "web.Request") -> "web.Response": if agent is not None: try: - agent.interrupt("Stop requested via API") + request_hard_interrupt(agent, "Stop requested via API") except Exception: pass # The stopped run is abandoned — reap only the background diff --git a/gateway/run.py b/gateway/run.py index 532ff9c2a942..5698c98716e0 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -59,6 +59,7 @@ ) from agent.conversation_loop import INTERRUPT_WAITING_FOR_MODEL_PREFIX from agent.i18n import t +from agent.interrupt_compat import request_hard_interrupt from hermes_cli.config import cfg_get from hermes_cli.fallback_config import get_fallback_chain @@ -9044,7 +9045,7 @@ def _interrupt_running_agents(self, reason: str) -> None: if agent is _AGENT_PENDING_SENTINEL: continue try: - agent.interrupt(reason) + request_hard_interrupt(agent, reason) logger.debug("Interrupted running agent for session %s during shutdown", session_key) except Exception as e: logger.debug("Failed interrupting agent during shutdown: %s", e) @@ -22289,7 +22290,7 @@ async def _interrupt_and_clear_session( _process_task_id = "" _process_baseline = None if running_agent and running_agent is not _AGENT_PENDING_SENTINEL: - running_agent.interrupt(interrupt_reason) + request_hard_interrupt(running_agent, interrupt_reason) _process_task_id = getattr( running_agent, "_gateway_turn_process_task_id", "" ) @@ -24529,8 +24530,8 @@ def _run_sync_with_timeout_lifecycle(): # Interrupt the agent if it's still running so the thread # pool worker is freed. - if _timed_out_agent and hasattr(_timed_out_agent, "interrupt"): - _timed_out_agent.interrupt(_INTERRUPT_REASON_TIMEOUT) + if _timed_out_agent: + request_hard_interrupt(_timed_out_agent, _INTERRUPT_REASON_TIMEOUT) _timeout_mins = int(_agent_timeout // 60) or 1 diff --git a/hermes_state.py b/hermes_state.py index 71a77587fb62..39ec0806fe84 100644 --- a/hermes_state.py +++ b/hermes_state.py @@ -3599,6 +3599,92 @@ def get_compression_failure_cooldown( "error": error, } + def get_compression_failure_cooldown_row( + self, + session_id: str, + ) -> Dict[str, Any]: + """Return the exact stored cooldown columns without expiry filtering. + + Compression cancellation uses this under its session lease so rollback + can preserve an expired row, a partially-null row, or an absent session + exactly instead of converting those states through the active-cooldown + API. + """ + if not session_id: + return {"session_exists": False, "cooldown_until": None, "error": None} + with self._lock: + row = self._conn.execute( + "SELECT compression_failure_cooldown_until, compression_failure_error " + "FROM sessions WHERE id = ?", + (session_id,), + ).fetchone() + if row is None: + return {"session_exists": False, "cooldown_until": None, "error": None} + cooldown_until = ( + row["compression_failure_cooldown_until"] + if isinstance(row, sqlite3.Row) + else row[0] + ) + error = ( + row["compression_failure_error"] + if isinstance(row, sqlite3.Row) + else row[1] + ) + return { + "session_exists": True, + "cooldown_until": ( + float(cooldown_until) if cooldown_until is not None else None + ), + "error": error, + } + + def restore_compression_failure_cooldown_row( + self, + session_id: str, + snapshot: Dict[str, Any], + ) -> None: + """Restore and verify an exact cooldown-row snapshot. + + Unlike the ordinary record/clear helpers, this transactional rollback + API deliberately propagates write and verification failures. A caller + must not report cancellation as mutation-free when compensation failed. + """ + expected_exists = bool(snapshot.get("session_exists", False)) + if not expected_exists: + actual = self.get_compression_failure_cooldown_row(session_id) + if actual.get("session_exists", False): + raise RuntimeError( + "cannot restore absent compression cooldown row: session now exists" + ) + return + + deadline = snapshot.get("cooldown_until") + error = snapshot.get("error") + + def _do(conn): + cursor = conn.execute( + "UPDATE sessions SET compression_failure_cooldown_until = ?, " + "compression_failure_error = ? WHERE id = ?", + (deadline, error, session_id), + ) + if cursor.rowcount != 1: + raise RuntimeError( + f"compression cooldown rollback session missing: {session_id}" + ) + + self._execute_write(_do) + actual = self.get_compression_failure_cooldown_row(session_id) + expected = { + "session_exists": True, + "cooldown_until": float(deadline) if deadline is not None else None, + "error": error, + } + if actual != expected: + raise RuntimeError( + f"compression cooldown rollback verification failed: " + f"expected={expected!r}, actual={actual!r}" + ) + def clear_compression_failure_cooldown(self, session_id: str) -> None: """Clear any persisted compression-failure cooldown for a session.""" if not session_id: diff --git a/run_agent.py b/run_agent.py index 9a6542925989..072730f23dee 100644 --- a/run_agent.py +++ b/run_agent.py @@ -115,6 +115,7 @@ def _session_source_for_agent(platform: Optional[str]) -> str: _get_proxy_for_base_url, ) from agent.iteration_budget import IterationBudget +from agent.interrupt_compat import request_hard_interrupt from hermes_cli.env_loader import load_hermes_dotenv @@ -2976,7 +2977,7 @@ def _save_session_log(self, messages: List[Dict[str, Any]] = None): logging.warning(f"Failed to save session log: {e}") - def interrupt(self, message: str = None) -> None: + def interrupt(self, message: Optional[str] = None, *, hard_cancel: bool = False) -> None: """ Request the agent to interrupt its current tool-calling loop. @@ -2989,6 +2990,9 @@ def interrupt(self, message: str = None) -> None: Args: message: Optional new message that triggered the interrupt. If provided, the agent will include this in its response context. + hard_cancel: Mark this as an explicit stop rather than a redirect or + incoming-message interrupt. Compression may honor this + atomic signal even while ordinary interrupts are masked. Example (CLI): # In a separate input thread: @@ -3002,15 +3006,41 @@ def interrupt(self, message: str = None) -> None: """ # A hard stop and redirect share one lock so /stop cannot race with an # accepted correction and accidentally turn itself into a retry. + def _admit_hard_cancel() -> None: + event = getattr(self, "_hard_interrupt_requested", None) + if event is None: + return + fence = vars(self).get("_active_compression_commit_fence") + cancel_before_commit = getattr( + type(fence), "cancel_before_commit", None + ) + if callable(cancel_before_commit): + try: + # This sets the Event while holding the same lock used by + # begin_commit(). If commit already won, it waits for that + # tracked mutation to finish before publishing the stop. + cancel_before_commit(fence, event) + return + except Exception: + logger.debug( + "Compression hard-cancel fence admission failed", + exc_info=True, + ) + event.set() + _redirect_lock = getattr(self, "_pending_redirect_lock", None) if _redirect_lock is not None: with _redirect_lock: self._interrupt_requested = True self._interrupt_message = message + if hard_cancel: + _admit_hard_cancel() self._pending_redirect = None else: self._interrupt_requested = True self._interrupt_message = message + if hard_cancel: + _admit_hard_cancel() self._pending_redirect = None # Codex app-server owns its model/tool loop and watches a private @@ -3073,12 +3103,26 @@ def interrupt(self, message: str = None) -> None: children_copy = list(self._active_children) for child in children_copy: try: - child.interrupt(message) + if hard_cancel: + request_hard_interrupt(child, message) + else: + child.interrupt(message) except Exception as e: logger.debug("Failed to propagate interrupt to child agent: %s", e) if not self.quiet_mode: print("\n⚡ Interrupt requested" + (f": '{message[:40]}...'" if message and len(message) > 40 else f": '{message}'" if message else "")) + def hard_interrupt(self, message: Optional[str] = None) -> None: + """Request an explicit stop while preserving ``interrupt()`` ABI. + + Frontends can feature-detect this method and fall back to the legacy + ``interrupt()`` signature for synthetic or third-party agents. + """ + # Deliberately bypass dynamic dispatch: subclasses written against the + # legacy interrupt(message=None) ABI may override interrupt without the + # newer keyword-only hard_cancel argument. + AIAgent.interrupt(self, message, hard_cancel=True) + def clear_interrupt(self, *, preserve_redirect: bool = False) -> bool: """Clear the interrupt request and per-thread tool signal. @@ -3093,6 +3137,7 @@ def clear_interrupt(self, *, preserve_redirect: bool = False) -> bool: return False self._interrupt_requested = False self._interrupt_message = None + getattr(self, "_hard_interrupt_requested", threading.Event()).clear() if not preserve_redirect: self._pending_redirect = None else: @@ -3100,6 +3145,7 @@ def clear_interrupt(self, *, preserve_redirect: bool = False) -> bool: return False self._interrupt_requested = False self._interrupt_message = None + getattr(self, "_hard_interrupt_requested", threading.Event()).clear() if not preserve_redirect: self._pending_redirect = None self._interrupt_thread_signal_pending = False @@ -6907,7 +6953,10 @@ def _compress_context( auto-compress abort. Auto-compress callers use the default ``force=False``. """ - from agent.conversation_compression import compress_context + from agent.conversation_compression import ( + CompressionCommitFence, + compress_context, + ) from agent.portal_tags import ( get_conversation_context, reset_conversation_context, @@ -6932,19 +6981,39 @@ def _compress_context( root = self._conversation_root_id() if root: token = set_conversation_context(root) - try: - return compress_context( - self, messages, system_message, - approx_tokens=approx_tokens, task_id=task_id, focus_topic=focus_topic, - force=force, - defer_context_engine_notification=defer_context_engine_notification, - commit_fence=commit_fence, + # Every AIAgent compression has a fence, including ordinary in-turn and + # manual paths. hard_interrupt() uses this exact instance to serialize + # cancel admission against begin_commit(). + active_fence = commit_fence or CompressionCommitFence() + # A single agent can receive overlapping automatic/manual entrypoints. + # Serialize fence publication so a waiter cannot replace the fence of + # the attempt currently generating/committing a summary. + fence_registration_lock = vars(self).setdefault( + "_compression_commit_fence_lock", threading.RLock() + ) + with fence_registration_lock: + missing_fence = object() + previous_fence = vars(self).get( + "_active_compression_commit_fence", missing_fence ) - finally: - # Restore whatever the caller had, so a compaction never leaks its - # tag into the surrounding scope. - if token is not None: - reset_conversation_context(token) + self._active_compression_commit_fence = active_fence + try: + return compress_context( + self, messages, system_message, + approx_tokens=approx_tokens, task_id=task_id, focus_topic=focus_topic, + force=force, + defer_context_engine_notification=defer_context_engine_notification, + commit_fence=active_fence, + ) + finally: + if previous_fence is missing_fence: + vars(self).pop("_active_compression_commit_fence", None) + else: + self._active_compression_commit_fence = previous_fence + # Restore whatever the caller had, so a compaction never leaks its + # tag into the surrounding scope. + if token is not None: + reset_conversation_context(token) def _set_tool_guardrail_halt(self, decision: ToolGuardrailDecision) -> None: """Record the first guardrail decision that should stop this turn.""" diff --git a/tests/agent/test_auxiliary_explicit_cancellation.py b/tests/agent/test_auxiliary_explicit_cancellation.py new file mode 100644 index 000000000000..991738af8aab --- /dev/null +++ b/tests/agent/test_auxiliary_explicit_cancellation.py @@ -0,0 +1,622 @@ +"""Deterministic cross-thread cancellation tests for compression aux transports.""" + +from __future__ import annotations + +import contextvars +import threading +import time +from types import SimpleNamespace +from typing import Any, Callable + +import pytest + +from agent import auxiliary_client as aux + + +class _BlockingStream: + def __init__(self, started: threading.Event) -> None: + self.started = started + self.closed = threading.Event() + + def __iter__(self): + self.started.set() + self.closed.wait(timeout=5) + raise RuntimeError("transport closed") + + def close(self) -> None: + self.closed.set() + + def get_final_message(self) -> Any: + self.started.set() + self.closed.wait(timeout=5) + raise RuntimeError("transport closed") + + +class _GenericCompletions: + def __init__(self, stream: _BlockingStream) -> None: + self.stream = stream + + def create(self, **_kwargs: Any) -> _BlockingStream: + return self.stream + + +class _GenericClient: + def __init__(self, stream: _BlockingStream) -> None: + self.chat = SimpleNamespace(completions=_GenericCompletions(stream)) + self.stream = stream + self.closed = threading.Event() + + def close(self) -> None: + self.closed.set() + self.stream.close() + + +class _CodexResponses: + def __init__(self, stream: _BlockingStream) -> None: + self.stream = stream + + def create(self, **_kwargs: Any) -> _BlockingStream: + return self.stream + + +class _CodexRealClient: + def __init__(self, stream: _BlockingStream) -> None: + self.responses = _CodexResponses(stream) + self.api_key = "test" + self.base_url = "https://example.test/codex" + self.stream = stream + self.closed = threading.Event() + + def close(self) -> None: + self.closed.set() + self.stream.close() + + +class _AnthropicStreamContext: + def __init__(self, stream: _BlockingStream) -> None: + self.stream = stream + + def __enter__(self) -> _BlockingStream: + return self.stream + + def __exit__(self, *_args: Any) -> None: + self.stream.close() + + +class _AnthropicMessages: + def __init__(self, stream: _BlockingStream) -> None: + self.stream_obj = stream + + def stream(self, **_kwargs: Any) -> _AnthropicStreamContext: + return _AnthropicStreamContext(self.stream_obj) + + +class _AnthropicRealClient: + def __init__(self, stream: _BlockingStream) -> None: + self.messages = _AnthropicMessages(stream) + self.stream = stream + self.closed = threading.Event() + + def close(self) -> None: + self.closed.set() + self.stream.close() + + +class _BedrockRuntimeClient: + def __init__(self, started: threading.Event, release: threading.Event) -> None: + self.started = started + self.release = release + self.closed = threading.Event() + + def converse(self, **_kwargs: Any) -> dict[str, Any]: + self.started.set() + self.release.wait(timeout=5) + return { + "output": { + "message": { + "role": "assistant", + "content": [{"text": "cancelled response"}], + } + }, + "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, + "stopReason": "end_turn", + } + + def close(self) -> None: + self.closed.set() + + +def _cancel_silent_request( + client: Any, + started: threading.Event, + invoke: Callable[[Any], Any], +) -> tuple[BaseException, float]: + cancel_event = threading.Event() + result: dict[str, BaseException] = {} + + def _worker() -> None: + try: + with aux.aux_interrupt_protection(cancel_event=cancel_event): + invoke(client) + except BaseException as exc: + result["exc"] = exc + + worker = threading.Thread(target=_worker, daemon=True) + worker.start() + assert started.wait(timeout=1), "request never entered its silent transport" + cancelled_at = time.monotonic() + cancel_event.set() + worker.join(timeout=1) + elapsed = time.monotonic() - cancelled_at + assert not worker.is_alive(), "explicit cancellation did not wake the silent request" + return result["exc"], elapsed + + +def _invoke_generic(client: Any) -> Any: + return aux._relay_sync_completion( + client, + {"model": "test", "messages": [], "timeout": 30}, + create=lambda request: aux._create_with_progress( + client, request, "compression", force_stream=True + ), + ) + + +def test_protected_silent_provider_is_isolated_and_raises_frozen_explicit_cancel() -> None: + started = threading.Event() + stream = _BlockingStream(started) + client = _GenericClient(stream) + + exc, elapsed = _cancel_silent_request(client, started, _invoke_generic) + + assert isinstance(exc, aux.AuxiliaryExplicitCancellation) + assert exc.cause == "explicit_host_cancel" + assert not client.closed.is_set() + assert elapsed < 0.75 + stream.close() # release the bounded daemon provider worker + + +def test_codex_silent_stream_is_isolated_without_closing_shared_client() -> None: + started = threading.Event() + stream = _BlockingStream(started) + real_client = _CodexRealClient(stream) + client = aux.CodexAuxiliaryClient(real_client, "gpt-test") + + exc, elapsed = _cancel_silent_request(client, started, _invoke_generic) + + assert isinstance(exc, aux.AuxiliaryExplicitCancellation) + assert not real_client.closed.is_set() + assert elapsed < 0.75 + stream.close() + + +def test_cancelled_codex_orphan_timeout_preserves_cached_shared_client() -> None: + """A cancelled Codex worker's delayed timer owns only its event stream.""" + owner_started = threading.Event() + + class _SilentOwnerStream: + def __init__(self) -> None: + self.closed = threading.Event() + + def __iter__(self): + owner_started.set() + self.closed.wait(timeout=5) + raise RuntimeError("owner stream closed") + + def close(self) -> None: + self.closed.set() + + class _SuccessStream: + def __iter__(self): + message = SimpleNamespace( + type="message", + content=[SimpleNamespace(type="output_text", text="ok")], + ) + return iter( + [ + SimpleNamespace(type="response.output_item.done", item=message), + SimpleNamespace( + type="response.completed", + response=SimpleNamespace( + status="completed", id="success", usage=None + ), + ), + ] + ) + + def close(self) -> None: + pass + + owner_stream = _SilentOwnerStream() + + class _SharedResponses: + def __init__(self, real_client: Any) -> None: + self.real_client = real_client + + def create(self, **kwargs: Any) -> Any: + if self.real_client.closed.is_set(): + raise RuntimeError("shared client was closed") + if kwargs["model"] == "owner": + return owner_stream + return _SuccessStream() + + class _SharedRealClient: + def __init__(self) -> None: + self.closed = threading.Event() + self.api_key = "test" + self.base_url = "https://example.test/codex" + self.responses = _SharedResponses(self) + + def close(self) -> None: + self.closed.set() + owner_stream.close() + + real_client = _SharedRealClient() + wrapper = aux.CodexAuxiliaryClient(real_client, "gpt-test") + cache_key = ("openai-codex", False, None, None, None) + cancel_event = threading.Event() + owner_outcome: dict[str, BaseException] = {} + + def _run_owner() -> None: + try: + with aux.aux_interrupt_protection(cancel_event=cancel_event): + aux._relay_sync_completion( + wrapper, + {"model": "owner", "messages": [], "timeout": 0.12}, + ) + except BaseException as exc: + owner_outcome["exc"] = exc + + with aux._client_cache_lock: + aux._client_cache.clear() + aux._client_cache[cache_key] = (wrapper, "gpt-test", None) + owner = threading.Thread(target=_run_owner, daemon=True) + try: + owner.start() + assert owner_started.wait(timeout=1) + cancel_event.set() + owner.join(timeout=1) + assert not owner.is_alive() + assert isinstance(owner_outcome["exc"], aux.AuxiliaryExplicitCancellation) + # A real frontend clears the reusable host Event when the next turn + # starts. The orphan must retain a frozen per-attempt cancellation cause. + cancel_event.clear() + + # A second user can use the shared client while the cancelled provider + # worker is still orphaned and its total-timeout timer is still armed. + assert not owner_stream.closed.is_set() + concurrent = aux._relay_sync_completion( + wrapper, + {"model": "concurrent", "messages": [], "timeout": 1}, + ) + assert concurrent.choices[0].message.content == "ok" + + # Let the orphan's real adapter timer fire. It may close the attempt's + # event stream to wake that worker, but never the process-shared client. + assert owner_stream.closed.wait(timeout=1) + time.sleep(0.03) + assert not real_client.closed.is_set() + with aux._client_cache_lock: + assert aux._client_cache[cache_key][0] is wrapper + + successive = aux._relay_sync_completion( + wrapper, + {"model": "successive", "messages": [], "timeout": 1}, + ) + assert successive.choices[0].message.content == "ok" + finally: + owner_stream.close() + with aux._client_cache_lock: + aux._client_cache.clear() + + +@pytest.mark.parametrize("winner", ["timeout", "cancel"]) +def test_codex_timeout_and_explicit_cancel_have_one_linearized_outcome( + winner: str, +) -> None: + """Timeout and explicit cancel can never produce a mixed owner/cleanup result.""" + timer_read_started = threading.Event() + allow_timer_read_return = threading.Event() + request_cancelled = threading.Event() + stream_started = threading.Event() + + class _RacingCancelSource: + def is_set(self) -> bool: + if winner == "timeout" and threading.current_thread().name.startswith( + "Thread-" + ): + # Take the timer's false snapshot, then hold it at the exact seam + # where the historical implementation could race owner polling. + was_set = request_cancelled.is_set() + timer_read_started.set() + assert allow_timer_read_return.wait(timeout=1) + return was_set + return request_cancelled.is_set() + + class _SilentStream: + def __init__(self) -> None: + self.closed = threading.Event() + + def __iter__(self): + stream_started.set() + self.closed.wait(timeout=5) + raise RuntimeError("stream closed") + + def close(self) -> None: + self.closed.set() + + stream = _SilentStream() + + class _RealClient: + def __init__(self) -> None: + self.api_key = "test" + self.base_url = "https://example.test/codex" + self.responses = SimpleNamespace(create=lambda **_kwargs: stream) + self.closed = threading.Event() + + def close(self) -> None: + self.closed.set() + stream.close() + + real_client: Any = _RealClient() + wrapper = aux.CodexAuxiliaryClient(real_client, "gpt-test") + owner_outcome: dict[str, BaseException] = {} + + def _run_owner() -> None: + try: + with aux.aux_interrupt_protection(cancel_event=_RacingCancelSource()): + aux._relay_sync_completion( + wrapper, + {"model": "owner", "messages": [], "timeout": 0.08}, + ) + except BaseException as exc: + owner_outcome["exc"] = exc + + owner = threading.Thread(target=_run_owner, name="race-owner", daemon=True) + owner.start() + assert stream_started.wait(timeout=1) + if winner == "timeout": + assert timer_read_started.wait(timeout=1) + request_cancelled.set() + allow_timer_read_return.set() + else: + request_cancelled.set() + owner.join(timeout=1) + + assert not owner.is_alive() + if winner == "timeout": + assert real_client.closed.is_set() + assert isinstance(owner_outcome["exc"], TimeoutError) + assert not isinstance(owner_outcome["exc"], aux.AuxiliaryExplicitCancellation) + else: + assert isinstance(owner_outcome["exc"], aux.AuxiliaryExplicitCancellation) + assert stream.closed.wait(timeout=1), "cancelled timer did not wake its stream" + assert not real_client.closed.is_set() + + +def test_anthropic_silent_stream_is_isolated_without_closing_shared_client() -> None: + started = threading.Event() + stream = _BlockingStream(started) + real_client = _AnthropicRealClient(stream) + client = aux.AnthropicAuxiliaryClient( + real_client, + "claude-test", + "test-key", + "https://api.anthropic.test", + ) + + exc, elapsed = _cancel_silent_request(client, started, _invoke_generic) + + assert isinstance(exc, aux.AuxiliaryExplicitCancellation) + assert not real_client.closed.is_set() + assert elapsed < 0.75 + stream.close() + + +def test_cancelled_attempt_does_not_close_or_fail_concurrent_shared_client_call( + monkeypatch, +) -> None: + a_started = threading.Event() + a_release = threading.Event() + b_started = threading.Event() + b_release = threading.Event() + closed = threading.Event() + + class _SharedCompletions: + def create(self, **kwargs: Any) -> Any: + if kwargs["model"] == "session-a": + a_started.set() + a_release.wait(timeout=5) + else: + b_started.set() + b_release.wait(timeout=5) + if closed.is_set(): + raise RuntimeError("shared client was closed") + return SimpleNamespace( + choices=[SimpleNamespace(message=SimpleNamespace(content="ok"))] + ) + + client = SimpleNamespace( + chat=SimpleNamespace(completions=_SharedCompletions()), + close=lambda: closed.set(), + ) + cancel_event = threading.Event() + outcomes: dict[str, Any] = {} + evictions: list[Any] = [] + monkeypatch.setattr( + aux, "_evict_cached_client_instance", lambda value: evictions.append(value) + ) + + def _session_a() -> None: + try: + with aux.aux_interrupt_protection(cancel_event=cancel_event): + aux._relay_sync_completion( + client, {"model": "session-a", "messages": [], "timeout": 30} + ) + except BaseException as exc: + outcomes["a"] = exc + + def _session_b() -> None: + try: + outcomes["b"] = aux._relay_sync_completion( + client, {"model": "session-b", "messages": [], "timeout": 30} + ) + except BaseException as exc: # pragma: no cover - asserted below + outcomes["b"] = exc + + a_thread = threading.Thread(target=_session_a, daemon=True) + b_thread = threading.Thread(target=_session_b, daemon=True) + a_thread.start() + b_thread.start() + assert a_started.wait(timeout=1) + assert b_started.wait(timeout=1) + cancel_event.set() + a_thread.join(timeout=1) + try: + assert not a_thread.is_alive() + assert isinstance(outcomes["a"], aux.AuxiliaryExplicitCancellation) + assert not closed.is_set() + assert evictions == [] + b_release.set() + b_thread.join(timeout=1) + assert not b_thread.is_alive() + assert not isinstance(outcomes["b"], BaseException) + assert outcomes["b"].choices[0].message.content == "ok" + finally: + a_release.set() + b_release.set() + + +def test_bedrock_silent_nonstream_request_is_isolated_without_close_wakeup() -> None: + from agent.bedrock_adapter import _bedrock_runtime_client_cache, reset_client_cache + + started = threading.Event() + release = threading.Event() + runtime_client = _BedrockRuntimeClient(started, release) + reset_client_cache() + _bedrock_runtime_client_cache["us-test-1"] = runtime_client + client = aux.BedrockAuxiliaryClient("us-test-1", "bedrock-test") + try: + exc, elapsed = _cancel_silent_request(client, started, _invoke_generic) + finally: + release.set() + reset_client_cache() + + assert isinstance(exc, aux.AuxiliaryExplicitCancellation) + assert not runtime_client.closed.is_set() + assert elapsed < 0.75 + + +def test_unprotected_sync_completion_stays_on_calling_thread() -> None: + caller = threading.get_ident() + observed: list[int] = [] + client = SimpleNamespace( + chat=SimpleNamespace( + completions=SimpleNamespace( + create=lambda **_kwargs: ( + observed.append(threading.get_ident()), + SimpleNamespace(choices=[]), + )[1] + ) + ) + ) + + aux._relay_sync_completion(client, {"model": "test", "messages": []}) + + assert observed == [caller] + + +def test_isolated_provider_worker_inherits_protection_and_progress_hook() -> None: + caller = threading.get_ident() + cancel_event = threading.Event() + progress: list[str] = [] + observed: dict[str, Any] = {} + + def _create(**_kwargs: Any) -> Any: + observed["thread"] = threading.get_ident() + observed["protected"] = aux._aux_interrupt_protected() + aux._notify_aux_progress() + return SimpleNamespace(choices=[]) + + client = SimpleNamespace( + chat=SimpleNamespace(completions=SimpleNamespace(create=_create)) + ) + with aux.aux_progress_hook(lambda: progress.append("tick")), aux.aux_interrupt_protection( + cancel_event=cancel_event + ): + aux._relay_sync_completion(client, {"model": "test", "messages": []}) + + assert observed["protected"] is True + assert observed["thread"] != caller + assert progress == ["tick"] + + +def test_isolated_provider_worker_inherits_caller_contextvars() -> None: + from tools.approval import ( + get_current_session_key, + reset_current_session_key, + set_current_session_key, + ) + + arbitrary = contextvars.ContextVar("isolated-provider-test", default="missing") + arbitrary_token = arbitrary.set("caller-value") + session_token = set_current_session_key("session-from-caller") + observed: dict[str, str] = {} + client = SimpleNamespace( + chat=SimpleNamespace( + completions=SimpleNamespace( + create=lambda **_kwargs: ( + observed.update( + arbitrary=arbitrary.get(), + session_key=get_current_session_key(), + ), + SimpleNamespace(choices=[]), + )[1] + ) + ) + ) + try: + with aux.aux_interrupt_protection(cancel_event=threading.Event()): + aux._relay_sync_completion(client, {"model": "test", "messages": []}) + finally: + reset_current_session_key(session_token) + arbitrary.reset(arbitrary_token) + + assert observed == { + "arbitrary": "caller-value", + "session_key": "session-from-caller", + } + + +def test_hard_cancel_wins_when_provider_result_is_published_in_same_race() -> None: + cancel_event = threading.Event() + + def _create(**_kwargs: Any) -> Any: + cancel_event.set() + return SimpleNamespace(choices=[]) + + client = SimpleNamespace( + chat=SimpleNamespace(completions=SimpleNamespace(create=_create)) + ) + with aux.aux_interrupt_protection(cancel_event=cancel_event): + with pytest.raises(aux.AuxiliaryExplicitCancellation): + aux._relay_sync_completion(client, {"model": "test", "messages": []}) + + +def test_unrelated_interrupted_error_is_not_reclassified_as_explicit_cancel() -> None: + client = SimpleNamespace( + chat=SimpleNamespace( + completions=SimpleNamespace( + create=lambda **_kwargs: (_ for _ in ()).throw( + InterruptedError("provider syscall interrupted") + ) + ) + ), + close=lambda: None, + ) + + with aux.aux_interrupt_protection(cancel_event=threading.Event()): + with pytest.raises(InterruptedError, match="provider syscall interrupted") as caught: + aux._relay_sync_completion(client, {"model": "test", "messages": []}) + + assert not isinstance(caught.value, aux.AuxiliaryExplicitCancellation) diff --git a/tests/agent/test_compression_concurrent_fork.py b/tests/agent/test_compression_concurrent_fork.py index 7afa6d6ddcea..7dc164c425d0 100644 --- a/tests/agent/test_compression_concurrent_fork.py +++ b/tests/agent/test_compression_concurrent_fork.py @@ -28,8 +28,10 @@ from __future__ import annotations +import copy import inspect import os +import sqlite3 import threading import time from pathlib import Path @@ -40,7 +42,12 @@ from hermes_state import SessionDB -def _build_agent_with_db(db: SessionDB, session_id: str): +def _build_agent_with_db( + db: SessionDB, + session_id: str, + *, + stub_compressor: bool = True, +): """Build an AIAgent that's wired to ``db`` and pinned to ``session_id``.""" with patch.dict(os.environ, {"OPENROUTER_API_KEY": "test-key"}): from run_agent import AIAgent @@ -60,6 +67,9 @@ def _build_agent_with_db(db: SessionDB, session_id: str): # an LLM call. Sleep inside compress() so the two threads' rotations # actually overlap — without that the OS could happen to serialize them # and hide the bug. + if not stub_compressor: + return agent + compressor = MagicMock() def _compress_with_overlap(*_a, **_kw): @@ -871,7 +881,504 @@ def test_lease_refresher_failure_window_is_bounded_by_ttl() -> None: ) +def test_hard_interrupt_aborts_compression_and_unblocks_session_writes(tmp_path: Path) -> None: + """Ctrl+C must abort an interrupt-protected summary without leaving the + session write-blocked behind its compression lease.""" + from agent import auxiliary_client as aux + + db = SessionDB(db_path=tmp_path / "state.db") + session_id = "HARD_INTERRUPT_COMPRESSION_TEST" + db.create_session(session_id, source="cli") + + agent = _build_agent_with_db(db, session_id) + messages = [{"role": "user", "content": f"m{i}"} for i in range(20)] + original_messages = copy.deepcopy(messages) + + def _cancelled_compress(*_args, **_kwargs): + agent._hard_interrupt_requested.set() + assert aux._aux_interrupt_cancel_requested() is True + messages[0]["content"] = "must be rolled back" + raise aux.AuxiliaryExplicitCancellation() + + agent.context_compressor.compress.side_effect = _cancelled_compress + + compressed, _prompt = agent._compress_context( + messages, "sys", approx_tokens=120_000 + ) + + assert compressed == original_messages + assert messages == original_messages + assert db.get_compression_lock_holder(session_id) is None + db.append_message(session_id, "assistant", "writes recovered") + + +def test_late_hard_interrupt_restores_full_compressor_attempt_state_and_retry( + tmp_path: Path, +) -> None: + """A stop after provider success but before compress() returns is a true no-op.""" + db = SessionDB(db_path=tmp_path / "state.db") + session_id = "LATE_HARD_INTERRUPT_STATE_TEST" + db.create_session(session_id, source="cli") + agent = _build_agent_with_db(db, session_id) + agent.compression_in_place = True + agent._cached_system_prompt = "sys" + messages = [{"role": "user", "content": f"m{i}"} for i in range(20)] + provider_returned = threading.Event() + allow_compress_return = threading.Event() + shared_telemetry = {"shared": [1, 2, 3]} + state_fields = { + "_previous_summary": "old-summary", + "_summary_has_user_turn": False, + "compression_count": 4, + "_last_compression_savings_pct": 37.5, + "_ineffective_compression_count": 1, + "_anti_thrash_recovery_deadline": 123.0, + "_fallback_compression_streak": 1, + "_verify_compaction_cleared_threshold": False, + "_last_compression_made_progress": False, + "_summary_failure_cooldown_until": 456.0, + "_cooldown_persist_failed": True, + "_last_summary_error": "old-error", + "_consecutive_timeout_failures": 2, + "_last_summary_dropped_count": 3, + "_last_summary_fallback_used": True, + "_last_compress_aborted": False, + "_last_summary_auth_failure": True, + "_last_summary_network_failure": True, + "_last_aux_model_failure_error": "old-aux-error", + "_last_aux_model_failure_model": "old-aux-model", + "_summary_model_fallen_back": True, + "summary_model": "old-summary-model", + "_last_compression_telemetry": shared_telemetry, + "_active_compression_telemetry": shared_telemetry, + "_compression_telemetry_seed": {"seed": [3]}, + } + for name, value in state_fields.items(): + setattr(agent.context_compressor, name, copy.deepcopy(value)) + restored_shared_telemetry = copy.deepcopy(shared_telemetry) + agent.context_compressor._last_compression_telemetry = restored_shared_telemetry + agent.context_compressor._active_compression_telemetry = restored_shared_telemetry + + def _provider_succeeded_then_waits(*_args, **_kwargs): + for name in state_fields: + setattr(agent.context_compressor, name, f"mutated-{name}") + provider_returned.set() + assert allow_compress_return.wait(timeout=5) + return [ + {"role": "user", "content": "[CONTEXT COMPACTION] cancelled summary"}, + {"role": "user", "content": "tail"}, + ] + + agent.context_compressor.compress.side_effect = _provider_succeeded_then_waits + result: dict[str, tuple] = {} + worker = threading.Thread( + target=lambda: result.setdefault( + "value", agent._compress_context(messages, "sys", approx_tokens=120_000) + ), + daemon=True, + ) + worker.start() + assert provider_returned.wait(timeout=2) + agent.hard_interrupt("cancel after provider return") + allow_compress_return.set() + worker.join(timeout=5) + + assert not worker.is_alive() + assert result["value"][0] is messages + assert { + name: copy.deepcopy(getattr(agent.context_compressor, name)) + for name in state_fields + } == state_fields + assert ( + agent.context_compressor._active_compression_telemetry + is agent.context_compressor._last_compression_telemetry + ) + assert db.get_compression_lock_holder(session_id) is None + + agent.clear_interrupt() + agent.context_compressor.compress.side_effect = lambda *_a, **_kw: [ + {"role": "user", "content": "[CONTEXT COMPACTION] retry summary"}, + {"role": "user", "content": "tail"}, + ] + retried, _prompt = agent._compress_context( + messages, "sys", approx_tokens=120_000 + ) + assert retried is not messages + assert retried[0]["content"] == "[CONTEXT COMPACTION] retry summary" + + +def test_force_cancel_restores_newer_durable_cooldown_captured_under_lease( + tmp_path: Path, +) -> None: + """A stale forced attempt rolls back to the lease-protected durable row.""" + from agent.auxiliary_client import AuxiliaryExplicitCancellation + from agent.context_compressor import ContextCompressor + + db = SessionDB(db_path=tmp_path / "state.db") + session_id = "FORCE_CANCEL_DURABLE_COOLDOWN" + db.create_session(session_id, source="cli") + + # B binds first and therefore has no local cooldown. A then persists a + # newer cooldown for the same durable session before B acquires its lease. + stale_agent = _build_agent_with_db( + db, session_id, stub_compressor=False + ) + writer_agent = _build_agent_with_db( + db, session_id, stub_compressor=False + ) + stale = stale_agent.context_compressor + writer = writer_agent.context_compressor + assert isinstance(stale, ContextCompressor) + assert isinstance(writer, ContextCompressor) + assert stale._summary_failure_cooldown_until == 0.0 + + writer._record_compression_failure_cooldown(120.0, "newer durable failure") + durable_before = tuple( + db._conn.execute( + "SELECT compression_failure_cooldown_until, compression_failure_error " + "FROM sessions WHERE id = ?", + (session_id,), + ).fetchone() + ) + assert durable_before[0] is not None + + stale_seed = {"seed": ["truly-pre-attempt"]} + stale._compression_telemetry_seed = copy.deepcopy(stale_seed) + stale._previous_summary = "pre-attempt-summary" + stale_agent._compression_feasibility_checked = True + stale_agent.compression_in_place = True + messages = [{"role": "user", "content": f"m{i}"} for i in range(20)] + + real_clear = ContextCompressor._clear_compression_failure_cooldown + + def _clear_then_hard_cancel() -> None: + real_clear(stale) + stale_agent._hard_interrupt_requested.set() + raise AuxiliaryExplicitCancellation() + + # Exercise the built-in force=True mutation point deterministically: force + # clears the durable cooldown, then the frozen host cancellation unwinds it. + stale._clear_compression_failure_cooldown = _clear_then_hard_cancel + + compressed, _prompt = stale_agent._compress_context( + messages, + "sys", + approx_tokens=120_000, + force=True, + ) + + assert compressed is messages + durable_after = tuple( + db._conn.execute( + "SELECT compression_failure_cooldown_until, compression_failure_error " + "FROM sessions WHERE id = ?", + (session_id,), + ).fetchone() + ) + assert durable_after == durable_before + assert stale._summary_failure_cooldown_until > time.monotonic() + assert stale._last_summary_error == "newer durable failure" + assert stale._cooldown_persist_failed is False + assert stale._compression_telemetry_seed == stale_seed + assert stale._previous_summary == "pre-attempt-summary" + assert db.get_compression_lock_holder(session_id) is None + + # A future compressor refresh must still observe the exact row rather than + # the cancelled force attempt having permanently cleared it. + future_agent = _build_agent_with_db( + db, session_id, stub_compressor=False + ) + future = future_agent.context_compressor.get_active_compression_failure_cooldown( + refresh=True + ) + assert future is not None + assert future["error"] == "newer durable failure" + + +def test_unrelated_interrupted_error_propagates_and_releases_compression_lease( + tmp_path: Path, +) -> None: + """A plugin/OS InterruptedError is a failure, not an explicit transaction abort.""" + db = SessionDB(db_path=tmp_path / "state.db") + session_id = "UNRELATED_INTERRUPT_COMPRESSION_TEST" + db.create_session(session_id, source="cli") + agent = _build_agent_with_db(db, session_id) + messages = [{"role": "user", "content": f"m{i}"} for i in range(20)] + + def _provider_interrupted(*_args, **_kwargs): + messages[0]["content"] = "must be rolled back" + raise InterruptedError("provider syscall interrupted") + agent.context_compressor.compress.side_effect = _provider_interrupted + with pytest.raises(InterruptedError, match="provider syscall interrupted"): + agent._compress_context(messages, "sys", approx_tokens=120_000) + + assert db.get_compression_lock_holder(session_id) is None + db.append_message(session_id, "assistant", "writes recovered") + + +def test_redirect_interrupt_remains_protected_during_compression(tmp_path: Path) -> None: + """Redirects use interrupt_requested=True/message=None; only the atomic + hard-cancel event may override summary protection.""" + from agent import auxiliary_client as aux + + db = SessionDB(db_path=tmp_path / "state.db") + session_id = "REDIRECT_COMPRESSION_TEST" + db.create_session(session_id, source="cli") + agent = _build_agent_with_db(db, session_id) + agent._interrupt_requested = True + agent._interrupt_message = None + agent._pending_redirect = "new correction" + messages = [{"role": "user", "content": f"m{i}"} for i in range(20)] + + def _protected_noop(current, **_kwargs): + assert aux._aux_interrupt_cancel_requested() is False + return copy.deepcopy(current) + + agent.context_compressor.compress.side_effect = _protected_noop + + compressed, _prompt = agent._compress_context( + messages, "sys", approx_tokens=120_000 + ) + assert compressed == messages + assert db.get_compression_lock_holder(session_id) is None + + +def test_hard_cancel_between_compress_return_and_commit_begin_wins_atomically( + tmp_path: Path, +) -> None: + """The hard-stop admission and commit admission share one fence lock.""" + db = SessionDB(db_path=tmp_path / "state.db") + session_id = "HARD_CANCEL_COMMIT_RACE" + db.create_session(session_id, source="tui") + agent = _build_agent_with_db(db, session_id) + agent.compression_in_place = True + agent._cached_system_prompt = "sys" + messages = [{"role": "user", "content": f"m{i}"} for i in range(20)] + before_commit = threading.Event() + allow_commit_check = threading.Event() + + class _CommitBarrierList(list): + def __eq__(self, other): + before_commit.set() + assert allow_commit_check.wait(timeout=5) + return super().__eq__(other) + + agent.context_compressor.compress.side_effect = lambda *_a, **_kw: _CommitBarrierList( + [ + {"role": "user", "content": "[CONTEXT COMPACTION] summary"}, + {"role": "user", "content": "tail"}, + ] + ) + archive_spy = MagicMock(wraps=db.archive_and_compact) + db.archive_and_compact = archive_spy + result: dict[str, tuple] = {} + worker = threading.Thread( + target=lambda: result.setdefault( + "value", agent._compress_context(messages, "sys", approx_tokens=120_000) + ), + daemon=True, + ) + worker.start() + assert before_commit.wait(timeout=2) + + agent.hard_interrupt("cancel before commit admission") + allow_commit_check.set() + worker.join(timeout=5) + + assert not worker.is_alive() + assert result["value"][0] is messages + archive_spy.assert_not_called() + assert db.get_compression_lock_holder(session_id) is None + + +def test_hard_stop_waits_for_commit_already_admitted(tmp_path: Path) -> None: + """A surfaced stop never races an untracked post-return transcript commit.""" + db = SessionDB(db_path=tmp_path / "state.db") + session_id = "HARD_CANCEL_AFTER_COMMIT_ADMISSION" + db.create_session(session_id, source="tui") + agent = _build_agent_with_db(db, session_id) + agent.compression_in_place = True + agent._cached_system_prompt = "sys" + messages = [{"role": "user", "content": f"m{i}"} for i in range(20)] + commit_started = threading.Event() + allow_commit = threading.Event() + stop_returned = threading.Event() + original_archive = db.archive_and_compact + + def _blocked_archive(*args, **kwargs): + commit_started.set() + assert allow_commit.wait(timeout=5) + return original_archive(*args, **kwargs) + + db.archive_and_compact = _blocked_archive + agent.context_compressor.compress.side_effect = lambda *_a, **_kw: [ + {"role": "user", "content": "[CONTEXT COMPACTION] summary"}, + {"role": "user", "content": "tail"}, + ] + compression_result: dict[str, tuple] = {} + compression = threading.Thread( + target=lambda: compression_result.setdefault( + "value", agent._compress_context(messages, "sys", approx_tokens=120_000) + ), + daemon=True, + ) + compression.start() + assert commit_started.wait(timeout=2) + + stop = threading.Thread( + target=lambda: ( + agent.hard_interrupt("stop after commit admission"), + stop_returned.set(), + ), + daemon=True, + ) + stop.start() + assert not stop_returned.wait(timeout=0.1) + allow_commit.set() + compression.join(timeout=5) + stop.join(timeout=5) + + assert not compression.is_alive() + assert not stop.is_alive() + assert stop_returned.is_set() + assert compression_result["value"][0][0]["content"] == ( + "[CONTEXT COMPACTION] summary" + ) + assert agent._hard_interrupt_requested.is_set() + assert db.get_compression_lock_holder(session_id) is None + + +@pytest.mark.parametrize("deadline_offset", [-10.0, 0.05, None]) +def test_force_cancel_restores_exact_expired_or_expiring_cooldown_row( + tmp_path: Path, + deadline_offset: float | None, +) -> None: + """Cancellation preserves raw cooldown columns even after their deadline.""" + from agent.auxiliary_client import AuxiliaryExplicitCancellation + from agent.context_compressor import ContextCompressor + + db = SessionDB(db_path=tmp_path / "state.db") + session_id = f"RAW_COOLDOWN_{deadline_offset}" + db.create_session(session_id, source="cli") + deadline = time.time() + deadline_offset if deadline_offset is not None else None + db.restore_compression_failure_cooldown_row( + session_id, + { + "session_exists": True, + "cooldown_until": deadline, + "error": "expired-but-exact", + }, + ) + before = db.get_compression_failure_cooldown_row(session_id) + + agent = _build_agent_with_db(db, session_id, stub_compressor=False) + compressor = agent.context_compressor + assert isinstance(compressor, ContextCompressor) + # A stale local persistence-failure marker must not suppress restoration + # once the raw durable row was captured authoritatively under the lease. + compressor._cooldown_persist_failed = True + agent._compression_feasibility_checked = True + agent.compression_in_place = True + messages = [{"role": "user", "content": f"m{i}"} for i in range(20)] + real_clear = ContextCompressor._clear_compression_failure_cooldown + + def _mutate_then_cancel() -> None: + real_clear(compressor) + if deadline_offset is not None and deadline_offset > 0: + assert deadline is not None + while time.time() <= deadline: + time.sleep(0.005) + agent._hard_interrupt_requested.set() + raise AuxiliaryExplicitCancellation() + + compressor._clear_compression_failure_cooldown = _mutate_then_cancel + + compressed, _prompt = agent._compress_context( + messages, + "sys", + approx_tokens=120_000, + force=True, + ) + + assert compressed is messages + assert db.get_compression_failure_cooldown_row(session_id) == before + assert db.get_compression_lock_holder(session_id) is None + + +def test_cooldown_rollback_failure_surfaces_and_releases_lease( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A failed compensating write cannot masquerade as a mutation-free cancel.""" + from agent.auxiliary_client import AuxiliaryExplicitCancellation + from agent.context_compressor import ContextCompressor + + db = SessionDB(db_path=tmp_path / "state.db") + session_id = "COOLDOWN_ROLLBACK_WRITE_FAILURE" + db.create_session(session_id, source="cli") + db.record_compression_failure_cooldown( + session_id, + time.time() + 120.0, + "must-restore", + ) + agent = _build_agent_with_db(db, session_id, stub_compressor=False) + compressor = agent.context_compressor + assert isinstance(compressor, ContextCompressor) + agent._compression_feasibility_checked = True + agent.compression_in_place = True + messages = [{"role": "user", "content": f"m{i}"} for i in range(20)] + real_clear = ContextCompressor._clear_compression_failure_cooldown + + def _mutate_then_cancel() -> None: + real_clear(compressor) + agent._hard_interrupt_requested.set() + raise AuxiliaryExplicitCancellation() + + compressor._clear_compression_failure_cooldown = _mutate_then_cancel + + def _rollback_write_fails(_self, _session_id, _snapshot) -> None: + raise sqlite3.OperationalError("forced rollback write failure") + + monkeypatch.setattr( + SessionDB, + "restore_compression_failure_cooldown_row", + _rollback_write_fails, + ) + + with pytest.raises(sqlite3.OperationalError, match="forced rollback write failure"): + agent._compress_context( + messages, + "sys", + approx_tokens=120_000, + force=True, + ) + + assert db.get_compression_lock_holder(session_id) is None + + +def test_exact_cooldown_restore_api_propagates_sqlite_write_failure( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + db = SessionDB(db_path=tmp_path / "state.db") + session_id = "RAW_COOLDOWN_WRITE_FAILURE" + db.create_session(session_id, source="test") + + def _write_fails(_callback) -> None: + raise sqlite3.OperationalError("forced low-level write failure") + + monkeypatch.setattr(db, "_execute_write", _write_fails) + + with pytest.raises(sqlite3.OperationalError, match="forced low-level write failure"): + db.restore_compression_failure_cooldown_row( + session_id, + { + "session_exists": True, + "cooldown_until": time.time() + 10.0, + "error": "must propagate", + }, + ) diff --git a/tests/agent/test_compression_interrupt_protection.py b/tests/agent/test_compression_interrupt_protection.py index 075630c108c7..7aa96f15bde7 100644 --- a/tests/agent/test_compression_interrupt_protection.py +++ b/tests/agent/test_compression_interrupt_protection.py @@ -14,6 +14,8 @@ from unittest.mock import patch +import pytest + import agent.auxiliary_client as aux @@ -38,6 +40,15 @@ def test_restores_on_exception(self): assert aux._aux_interrupt_protected() is False + def test_nested_protection_preserves_explicit_cancel_check(self): + """A hard-cancel hook installed by the compression host survives the + compressor's nested protection scope.""" + with aux.aux_interrupt_protection(cancel_check=lambda: True): + with aux.aux_interrupt_protection(): + assert aux._aux_interrupt_protected() is True + assert aux._aux_interrupt_cancel_requested() is True + assert aux._aux_interrupt_cancel_requested() is False + class TestCompressionProtectsSummaryCall: """The compressor must wrap its summary call_llm in aux_interrupt_protection @@ -82,3 +93,83 @@ def fake_call_llm(**kwargs): ) # Protection must be cleared after the call returns. assert aux._aux_interrupt_protected() is False + + def test_explicit_interrupt_is_not_degraded_into_summary_fallback(self): + """Ctrl+C cancellation must escape summary fallback so the outer + compression transaction can abort without rotating the session.""" + from agent.context_compressor import ContextCompressor + + with patch("agent.context_compressor.get_model_context_length", return_value=100000): + c = ContextCompressor(model="test", quiet_mode=True) + + msgs = [ + {"role": "user", "content": "do a thing"}, + {"role": "assistant", "content": "working"}, + {"role": "user", "content": "more"}, + {"role": "assistant", "content": "done"}, + ] + with aux.aux_interrupt_protection(cancel_check=lambda: True), patch( + "agent.context_compressor.call_llm", + side_effect=aux.AuxiliaryExplicitCancellation(), + ): + try: + c._generate_summary(msgs) + except aux.AuxiliaryExplicitCancellation as exc: + assert exc.cause == "explicit_host_cancel" + else: + raise AssertionError("compression swallowed an explicit interrupt") + + def test_non_explicit_interrupted_error_remains_provider_failure(self): + """An unrelated provider/OS InterruptedError must keep the established + summary-failure fallback semantics when no host cancel was requested.""" + from agent.context_compressor import ContextCompressor + + with patch("agent.context_compressor.get_model_context_length", return_value=100000): + c = ContextCompressor(model="test", quiet_mode=True) + + msgs = [ + {"role": "user", "content": "do a thing"}, + {"role": "assistant", "content": "working"}, + {"role": "user", "content": "more"}, + {"role": "assistant", "content": "done"}, + ] + with patch( + "agent.context_compressor.call_llm", + side_effect=InterruptedError("provider syscall interrupted"), + ): + assert c._generate_summary(msgs) is None + + def test_explicit_interrupt_restores_rehydration_state(self): + """Cancellation after the handoff scan must be a compressor no-op.""" + from agent.context_compressor import ContextCompressor + + with patch("agent.context_compressor.get_model_context_length", return_value=100000): + c = ContextCompressor( + model="test", + protect_first_n=1, + protect_last_n=1, + quiet_mode=True, + ) + c._previous_summary = "foreign-session-summary" + c._summary_has_user_turn = False + msgs = [ + {"role": "system", "content": "system"}, + {"role": "user", "content": "one"}, + {"role": "assistant", "content": "two"}, + {"role": "user", "content": "three"}, + {"role": "assistant", "content": "four"}, + {"role": "user", "content": "five"}, + {"role": "assistant", "content": "six"}, + {"role": "user", "content": "tail"}, + ] + + with patch.object( + c, + "_generate_summary", + side_effect=aux.AuxiliaryExplicitCancellation(), + ): + with pytest.raises(aux.AuxiliaryExplicitCancellation): + c.compress(msgs) + + assert c._previous_summary == "foreign-session-summary" + assert c._summary_has_user_turn is False diff --git a/tests/agent/test_interrupt_compat.py b/tests/agent/test_interrupt_compat.py new file mode 100644 index 000000000000..5b62f18e1935 --- /dev/null +++ b/tests/agent/test_interrupt_compat.py @@ -0,0 +1,103 @@ +"""Compatibility contract for explicit hard-stop producers.""" + +from __future__ import annotations + +import threading +from unittest.mock import MagicMock + +from agent.interrupt_compat import request_hard_interrupt + + +class _ModernAgent: + def __init__(self) -> None: + self.calls: list[tuple[str, str | None]] = [] + + def hard_interrupt(self, message: str | None = None) -> None: + self.calls.append(("hard", message)) + + def interrupt(self, message: str | None = None) -> None: + self.calls.append(("soft", message)) + + +class _LegacyAgent: + def __init__(self) -> None: + self.calls: list[tuple[str, str | None]] = [] + + def interrupt(self, message: str | None = None) -> None: + self.calls.append(("legacy", message)) + + +def test_explicit_producer_prefers_feature_detected_hard_interrupt() -> None: + agent = _ModernAgent() + + assert request_hard_interrupt(agent, "stop now") is True + + assert agent.calls == [("hard", "stop now")] + + +def test_explicit_producer_falls_back_to_old_interrupt_signature() -> None: + agent = _LegacyAgent() + + assert request_hard_interrupt(agent, "stop now") is True + + assert agent.calls == [("legacy", "stop now")] + + +def test_explicit_producer_reports_unsupported_agent() -> None: + assert request_hard_interrupt(object(), "stop now") is False + + +def test_dynamic_proxy_does_not_fabricate_hard_interrupt_support() -> None: + agent = MagicMock() + + assert request_hard_interrupt(agent, "stop now") is True + + agent.interrupt.assert_called_once_with("stop now") + agent.hard_interrupt.assert_not_called() + + +def test_inherited_hard_interrupt_bypasses_legacy_subclass_override() -> None: + from run_agent import AIAgent + + class LegacySubclass(AIAgent): + def __init__(self) -> None: + self.legacy_calls: list[str | None] = [] + self._hard_interrupt_requested = threading.Event() + self._pending_redirect_lock = threading.RLock() + self._pending_redirect = None + self._execution_thread_id = None + self._interrupt_thread_signal_pending = False + self._tool_worker_threads: set[int] = set() + self._tool_worker_threads_lock = threading.Lock() + self._active_children: list[object] = [] + self._active_children_lock = threading.Lock() + self.quiet_mode = True + self.api_mode = "test" + + def interrupt(self, message: str | None = None) -> None: # type: ignore[override] + self.legacy_calls.append(message) + + agent = LegacySubclass() + + assert request_hard_interrupt(agent, "stop now") is True + + assert agent.legacy_calls == [] + assert agent._hard_interrupt_requested.is_set() + assert agent._interrupt_requested is True + assert agent._interrupt_message == "stop now" + + +def test_tui_subagent_interrupt_is_an_explicit_hard_stop() -> None: + import tools.delegate_tool as delegate_tool + + agent = _ModernAgent() + subagent_id = "sa-hard-stop-test" + with delegate_tool._active_subagents_lock: + delegate_tool._active_subagents[subagent_id] = {"agent": agent} + try: + assert delegate_tool.interrupt_subagent(subagent_id) is True + finally: + with delegate_tool._active_subagents_lock: + delegate_tool._active_subagents.pop(subagent_id, None) + + assert agent.calls == [("hard", f"Interrupted via TUI ({subagent_id})")] diff --git a/tests/agent/test_subagent_lifecycle.py b/tests/agent/test_subagent_lifecycle.py index ae1aa5a73fda..c32241fa8f32 100644 --- a/tests/agent/test_subagent_lifecycle.py +++ b/tests/agent/test_subagent_lifecycle.py @@ -24,9 +24,15 @@ def __init__(self, ident="sa-test"): self.provider = "test" self.model = "test-model" self.interrupted = False + self.interrupt_kind = None def interrupt(self, _reason): self.interrupted = True + self.interrupt_kind = "soft" + + def hard_interrupt(self, _reason): + self.interrupted = True + self.interrupt_kind = "hard" @pytest.fixture @@ -76,6 +82,17 @@ def test_cancel_is_cooperative_and_forged_handle_is_unknown(lifecycle): assert other_service.status(handle).state is SubagentState.UNKNOWN +def test_cancel_uses_explicit_hard_interrupt(lifecycle): + handle = lifecycle.launch(SubagentLaunchRequest(goal="x")) + record = lifecycle._record(handle) + assert record is not None and record.agent is not None + + assert lifecycle.cancel(handle, reason="explicit user cancel").accepted + + assert record.agent.interrupt_kind == "hard" + lifecycle.wait(handle, timeout_seconds=1) + + diff --git a/tests/run_agent/test_interrupt_propagation.py b/tests/run_agent/test_interrupt_propagation.py index 7e3085f1d2c0..f53afe71864c 100644 --- a/tests/run_agent/test_interrupt_propagation.py +++ b/tests/run_agent/test_interrupt_propagation.py @@ -27,6 +27,7 @@ def _make_bare_agent(self): agent = AIAgent.__new__(AIAgent) agent._interrupt_requested = False agent._interrupt_message = None + agent._hard_interrupt_requested = threading.Event() agent._execution_thread_id = None agent._interrupt_thread_signal_pending = False agent._active_children = [] @@ -54,6 +55,38 @@ def test_parent_interrupt_sets_child_flag(self): assert is_interrupted() is False assert parent._interrupt_thread_signal_pending is True + def test_hard_cancel_is_explicit_atomic_and_propagated(self): + parent = self._make_bare_agent() + child = self._make_bare_agent() + parent._active_children.append(child) + + parent.interrupt("Stop requested", hard_cancel=True) + + assert parent._hard_interrupt_requested.is_set() + assert child._hard_interrupt_requested.is_set() + parent.clear_interrupt() + assert not parent._hard_interrupt_requested.is_set() + + def test_message_interrupt_does_not_set_hard_cancel(self): + agent = self._make_bare_agent() + + agent.interrupt("new user message") + + assert agent._interrupt_requested is True + assert not agent._hard_interrupt_requested.is_set() + + def test_active_turn_redirect_does_not_set_hard_cancel(self): + agent = self._make_bare_agent() + agent._model_request_active = threading.Event() + agent._model_request_active.set() + agent._pending_redirect = None + + assert agent.redirect("new correction") is True + + assert agent._interrupt_requested is True + assert agent._interrupt_message is None + assert not agent._hard_interrupt_requested.is_set() + def test_child_clear_interrupt_at_start_clears_thread(self): """child.clear_interrupt() at start of run_conversation clears the bound execution thread's interrupt flag. diff --git a/tests/tui_gateway/test_compute_host.py b/tests/tui_gateway/test_compute_host.py index eb748a593611..fa0019722f13 100644 --- a/tests/tui_gateway/test_compute_host.py +++ b/tests/tui_gateway/test_compute_host.py @@ -6,6 +6,10 @@ import threading from pathlib import Path +import pytest + +from tui_gateway.compute_host import ComputeHost, HostSession + def _stdout_queue(proc: subprocess.Popen) -> queue.Queue[dict]: out: queue.Queue[dict] = queue.Queue() @@ -83,3 +87,42 @@ def test_compute_host_line_json_seed_turn_interrupt(): finally: if proc.poll() is None: proc.kill() + + +@pytest.mark.parametrize("kind", ["legacy", "hard-only", "dynamic-getattr"]) +def test_compute_host_interrupt_uses_explicit_stop_compatibility(kind): + calls = [] + + class _Legacy: + def interrupt(self): + calls.append("legacy") + + class _HardOnly: + def hard_interrupt(self): + calls.append("hard") + + class _Dynamic: + def interrupt(self): + calls.append("legacy") + + def __getattr__(self, name): + if name == "hard_interrupt": + return lambda: calls.append("fabricated-hard") + raise AttributeError(name) + + agent = { + "legacy": _Legacy(), + "hard-only": _HardOnly(), + "dynamic-getattr": _Dynamic(), + }[kind] + host = ComputeHost(heartbeat_secs=0) + host._sessions["s1"] = HostSession(sid="s1", agent=agent) + emitted = [] + host.emit = emitted.append + try: + host._handle_interrupt({"sid": "s1", "request_id": "stop"}) + finally: + host.close() + + assert calls == ["hard" if kind == "hard-only" else "legacy"] + assert emitted[-1]["applied"] is True diff --git a/tests/tui_gateway/test_protocol.py b/tests/tui_gateway/test_protocol.py index 23f285523102..00678f4b79b3 100644 --- a/tests/tui_gateway/test_protocol.py +++ b/tests/tui_gateway/test_protocol.py @@ -118,7 +118,54 @@ def test_err_envelope(server): } -# ── write_json ─────────────────────────────────────────────────────── +@pytest.mark.parametrize("kind", ["legacy", "hard-only", "dynamic-getattr"]) +def test_session_interrupt_uses_explicit_stop_compatibility(server, monkeypatch, kind): + calls = [] + + class _Legacy: + def interrupt(self): + calls.append("legacy") + + class _HardOnly: + def hard_interrupt(self): + calls.append("hard") + + class _Dynamic: + def interrupt(self): + calls.append("legacy") + + def __getattr__(self, name): + if name == "hard_interrupt": + return lambda: calls.append("fabricated-hard") + raise AttributeError(name) + + agent = { + "legacy": _Legacy(), + "hard-only": _HardOnly(), + "dynamic-getattr": _Dynamic(), + }[kind] + session = { + "agent": agent, + "history_lock": threading.Lock(), + "running": True, + "queued_prompt": "later", + "session_key": "session-key", + "_run_thread": None, + } + monkeypatch.setattr(server, "_tts_stream_stop", lambda: None) + monkeypatch.setattr(server, "_sess_nowait", lambda _params, _rid: (session, None)) + monkeypatch.setattr(server, "_sess", lambda _params, _rid: (session, None)) + monkeypatch.setattr(server, "_session_uses_compute_host", lambda _session: False) + monkeypatch.setattr(server, "_clear_pending", lambda _sid: None) + response = server._methods["session.interrupt"]( + "stop", {"session_id": "ui-session"} + ) + + assert response["result"]["status"] == "interrupted" + assert calls == ["hard" if kind == "hard-only" else "legacy"] + + +# ── write_json ──────────────────────────────────────────────── def test_write_json(capture): @@ -552,13 +599,13 @@ def test_skin_live_switch_end_to_end(server, tmp_path, monkeypatch): monkeypatch.setattr(server, "_emit", lambda ev, sid, payload=None: emitted.append((ev, payload))) # Baseline (default) — seeds the signature. - (tmp_path / "config.yaml").write_text("display:\n skin: default\n") + (tmp_path / "config.yaml").write_text("display:\n skin: default\n", encoding="utf-8") server._broadcast_skin_if_changed() emitted.clear() # Activate midnight, as `hermes config set display.skin midnight` would. time.sleep(0.01) # ensure the config mtime moves - (tmp_path / "config.yaml").write_text("display:\n skin: midnight\n") + (tmp_path / "config.yaml").write_text("display:\n skin: midnight\n", encoding="utf-8") server._broadcast_skin_if_changed() assert [ev for ev, _ in emitted] == ["skin.changed"] diff --git a/tools/delegate_tool.py b/tools/delegate_tool.py index e75727e602c8..18e47d429db1 100644 --- a/tools/delegate_tool.py +++ b/tools/delegate_tool.py @@ -33,6 +33,7 @@ from urllib.parse import urlsplit, urlunsplit from toolsets import TOOLSETS +from agent.interrupt_compat import request_hard_interrupt # Sentinel value used by the runtime provider system for providers that are # not natively known (named custom providers, third-party aggregators, etc.). @@ -196,7 +197,8 @@ def interrupt_subagent(subagent_id: str) -> bool: if agent is None: return False try: - agent.interrupt(f"Interrupted via TUI ({subagent_id})") + if not request_hard_interrupt(agent, f"Interrupted via TUI ({subagent_id})"): + return False except Exception as exc: logger.debug("interrupt_subagent(%s) failed: %s", subagent_id, exc) return False @@ -2193,9 +2195,8 @@ def _run_with_thread_capture(): except Exception as _timeout_exc: # Signal the child to stop so its thread can exit cleanly. try: - if hasattr(child, "interrupt"): - child.interrupt() - elif hasattr(child, "_interrupt_requested"): + interrupted = child is not None and request_hard_interrupt(child) + if not interrupted and child is not None and hasattr(child, "_interrupt_requested"): child._interrupt_requested = True except Exception: pass @@ -3275,9 +3276,8 @@ def _batch_runner(): def _batch_interrupt(): for _c in _child_agents: try: - if hasattr(_c, "interrupt"): - _c.interrupt("Async delegation cancelled") - elif hasattr(_c, "_interrupt_requested"): + interrupted = request_hard_interrupt(_c, "Async delegation cancelled") + if not interrupted and hasattr(_c, "_interrupt_requested"): _c._interrupt_requested = True except Exception: pass diff --git a/tui_gateway/compute_host.py b/tui_gateway/compute_host.py index 83aaf740b924..1f255533bd7b 100644 --- a/tui_gateway/compute_host.py +++ b/tui_gateway/compute_host.py @@ -21,6 +21,8 @@ from pathlib import Path from typing import Any, Callable +from agent.interrupt_compat import request_hard_interrupt + def now_ns() -> int: return time.perf_counter_ns() @@ -37,7 +39,7 @@ class SpikeAgent: def clear_interrupt(self) -> None: self._interrupt.clear() - def interrupt(self) -> None: + def interrupt(self, *, hard_cancel: bool = False) -> None: self._interrupt.set() def run_conversation( @@ -257,7 +259,7 @@ def _handle_interrupt(self, frame: dict[str, Any]) -> None: sid = str(frame.get("sid") or "") spike = self._sessions.get(sid) if spike is not None: - spike.agent.interrupt() + request_hard_interrupt(spike.agent) self.emit( { "type": "interrupt.ack", @@ -276,8 +278,8 @@ def _handle_interrupt(self, frame: dict[str, Any]) -> None: self.emit({"type": "interrupt.ack", "sid": sid, "request_id": frame.get("request_id"), "applied": False}) return agent = session.get("agent") - if agent is not None and hasattr(agent, "interrupt"): - agent.interrupt() + if agent is not None: + request_hard_interrupt(agent) with session.get("history_lock", threading.Lock()): session["_turn_cancel_requested"] = True session["queued_prompt"] = None diff --git a/tui_gateway/methods_session.py b/tui_gateway/methods_session.py index 44ef31ca82d5..91ff2a6946c7 100644 --- a/tui_gateway/methods_session.py +++ b/tui_gateway/methods_session.py @@ -2749,8 +2749,10 @@ def _(rid, params: dict) -> dict: session["queued_prompt"] = None session.pop("queued_prompts", None) session["_queued_prompt_generation"] = int(session.get("_queued_prompt_generation", 0)) + 1 - if should_interrupt and hasattr(session["agent"], "interrupt"): - session["agent"].interrupt() + if should_interrupt: + from agent.interrupt_compat import request_hard_interrupt + + request_hard_interrupt(session["agent"]) if not run_thread_alive: with session["history_lock"]: if session.get("running"): From 4ea379ca2e08b56a590e35b328de4fb4d74e2b6d Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Sun, 2 Aug 2026 21:50:20 +0530 Subject: [PATCH 10/69] fix(gateway): timed-out turn abandonment is an explicit hard stop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _abandon_timed_out_gateway_turn landed on main (eb4772ec2) after this PR's base and still used the soft interrupt(). Every other inactivity- timeout surface in this change (cron, gateway executor poll, delegate children) treats a timeout as an explicit stop that may cancel a protected compression summary — widen the same fix to this sibling. --- gateway/run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gateway/run.py b/gateway/run.py index 5698c98716e0..adb867f80d2d 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -2780,9 +2780,9 @@ def _abandon_timed_out_gateway_turn( timeout_fired.set() agent = agent_holder[0] if agent_holder else None - if agent is not None and hasattr(agent, "interrupt"): + if agent is not None: try: - agent.interrupt(_INTERRUPT_REASON_TIMEOUT) + request_hard_interrupt(agent, _INTERRUPT_REASON_TIMEOUT) except Exception: logger.debug("Timed-out agent interrupt failed", exc_info=True) From de0ce24c2e5660bf74d69cfd888e91d648cae54d Mon Sep 17 00:00:00 2001 From: ZachariahChu <1762459322@qq.com> Date: Sun, 26 Jul 2026 20:29:45 +0800 Subject: [PATCH 11/69] perf(cli): cap /model picker custom-endpoint probe at 1.5s The interactive /model picker probes the current custom endpoint live via fetch_api_models(), which defaults to a 5s timeout. A slow or flaky custom endpoint blocks the picker for up to 5s on open. The lmstudio picker probe already uses a 1.5s timeout for exactly this reason; apply the same fast-fail bound to the three custom-provider probe sites, gated on for_picker so the non-picker (5s) path is unchanged. --- hermes_cli/model_switch.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/hermes_cli/model_switch.py b/hermes_cli/model_switch.py index ef33260e2d5d..252986ed30ae 100644 --- a/hermes_cli/model_switch.py +++ b/hermes_cli/model_switch.py @@ -2702,6 +2702,7 @@ def _has_aws_sdk_creds_for_listing(slug: str) -> bool: live_models = fetch_api_models( api_key, api_url, + timeout=1.5 if for_picker else 5.0, # picker: fail fast so a slow custom endpoint doesn't block /model headers=_extra_headers_from_config(ep_cfg) or None, ) if live_models: @@ -2769,7 +2770,11 @@ def _has_aws_sdk_creds_for_listing(slug: str) -> bool: try: from hermes_cli.models import fetch_api_models - _live_models = fetch_api_models("", str(current_base_url).strip().rstrip("/")) + _live_models = fetch_api_models( + "", + str(current_base_url).strip().rstrip("/"), + timeout=1.5 if for_picker else 5.0, # picker: fail fast on a slow current endpoint + ) if _live_models: _models = _live_models except Exception: @@ -3012,6 +3017,7 @@ def _has_aws_sdk_creds_for_listing(slug: str) -> bool: live_models = fetch_api_models( api_key, api_url, + timeout=1.5 if for_picker else 5.0, # picker: fail fast so a slow custom endpoint doesn't block /model headers=grp.get("extra_headers") or None, ) if live_models: From 45c15d33e5ed7f42c6b4778ab8d34b40ad27cc86 Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Sun, 2 Aug 2026 22:01:42 +0530 Subject: [PATCH 12/69] test: update fetch_api_models call-shape assertions for explicit timeout Three tests pin the exact kwargs of the picker probe call (test_model_switch_custom_providers + two in test_user_providers_model_switch, the latter caught by CI slice 2); the picker-timeout change now always passes timeout explicitly (5.0 on the non-picker path), so the pinned shapes gain the key. --- tests/hermes_cli/test_model_switch_custom_providers.py | 2 +- tests/hermes_cli/test_user_providers_model_switch.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/hermes_cli/test_model_switch_custom_providers.py b/tests/hermes_cli/test_model_switch_custom_providers.py index e1eef0bc5a39..5b3513dc82dc 100644 --- a/tests/hermes_cli/test_model_switch_custom_providers.py +++ b/tests/hermes_cli/test_model_switch_custom_providers.py @@ -640,7 +640,7 @@ def fake_fetch_api_models(api_key, base_url, **kwargs): assert gateway_prov is not None, "Custom provider group not found in results" assert calls == [ - ("sk-gateway-key", "https://gateway.example.com/v1", {"headers": None}) + ("sk-gateway-key", "https://gateway.example.com/v1", {"timeout": 5.0, "headers": None}) ], "fetch_api_models must be called with the custom provider's credentials" assert gateway_prov["models"] == [ "gateway-model-a", diff --git a/tests/hermes_cli/test_user_providers_model_switch.py b/tests/hermes_cli/test_user_providers_model_switch.py index 46df5c0baeba..36c1d6d13cf4 100644 --- a/tests/hermes_cli/test_user_providers_model_switch.py +++ b/tests/hermes_cli/test_user_providers_model_switch.py @@ -158,7 +158,7 @@ def fake_fetch_api_models(api_key, base_url, **kwargs): ) assert user_prov is not None - assert calls == [("sk-test", "http://127.0.0.1:3000/api/v1", {"headers": None})] + assert calls == [("sk-test", "http://127.0.0.1:3000/api/v1", {"timeout": 5.0, "headers": None})] assert user_prov["models"] == ["old-configured-model", "new-live-model"] assert user_prov["total_models"] == 2 @@ -497,7 +497,7 @@ def _fake_fetch(api_key, api_url, **kwargs): assert probed.get("called") is True, "no-key bare endpoint should be probed" assert probed["api_key"] == "" - assert probed["kwargs"] == {"headers": None} + assert probed["kwargs"] == {"timeout": 5.0, "headers": None} row = next(p for p in providers if p["slug"] == "local-llamacpp") assert row["models"] == ["live-model-1", "live-model-2", "live-model-3"] assert row["total_models"] == 3 From fe497d8722eb43780ada1185db6d4012f65de885 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 2 Aug 2026 01:07:00 -0700 Subject: [PATCH 13/69] test(update): make EOL-churn dirtiness deterministic under racy-git stat caching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test_churn_across_more_files_than_fit_in_one_argv (e65ff9625f) asserts all 1200 checked-out files read dirty before normalization. Whether git diff content-compares an entry (seeing the CRLF churn) or trusts the stat cache depends on racy-git detection: entries whose recorded stat is non-racy (mtime older than the index write) read CLEAN. On CI a 1200-file checkout straddles that boundary nondeterministically — observed 92/1200 and 661/1200 dirty on two unrelated PRs within minutes (runs 30738759530, 30738842393). Empirically reproduced: freezing a non-racy stat cache gives 0/N dirty; bumping worktree mtimes past the index write forces content comparison and gives N/N deterministically. Fix: bump every worktree mtime after checkout in _managed_repo so all entries are stat-stale. Affects only the fixture; the production _normalize_managed_eol path is untouched. --- tests/hermes_cli/test_update_eol_churn.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/hermes_cli/test_update_eol_churn.py b/tests/hermes_cli/test_update_eol_churn.py index fb742d8001dd..a8fe2f762f51 100644 --- a/tests/hermes_cli/test_update_eol_churn.py +++ b/tests/hermes_cli/test_update_eol_churn.py @@ -59,6 +59,19 @@ def _managed_repo(tmp_path: Path, files: dict[str, bytes]) -> Path: for name in files: (repo / name).unlink() _git(repo, "checkout", "--", ".") + # Deterministic dirtiness: whether `git diff` content-checks an entry (and + # so sees the CRLF churn) or trusts the stat cache depends on racy-git + # detection — entries whose mtime equals the index timestamp get content- + # compared, later ones read clean. On a fast runner a large checkout + # straddles that boundary nondeterministically (CI flake: 92/661 of 1200 + # dirty). Bump every worktree mtime past the index write so ALL entries + # are stat-stale and git must content-compare each one. + import os as _os + import time as _time + + bumped = _time.time() + 5 + for name in files: + _os.utime(repo / name, (bumped, bumped)) return repo From 696fae8e78c0c93fe4ab1bdbaf7c56127ccfb11e Mon Sep 17 00:00:00 2001 From: Fangliquan Date: Thu, 23 Jul 2026 19:29:59 +0800 Subject: [PATCH 14/69] fix(agent): strip UTF-8 BOM when loading profile .env into secret scope Windows editors often save .env with a leading BOM; plain utf-8 left U+FEFF on the first key so multiplex get_secret missed that credential. --- agent/secret_scope.py | 6 +++++- tests/agent/test_secret_scope.py | 27 +++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/agent/secret_scope.py b/agent/secret_scope.py index 8b376d5fceff..1e4b30e3a70d 100644 --- a/agent/secret_scope.py +++ b/agent/secret_scope.py @@ -184,10 +184,14 @@ def load_env_file(env_path: Path) -> Dict[str, str]: ``set_secret_scope``. Mirrors python-dotenv's basic parsing (KEY=VALUE, ``export`` prefix, ``#`` comments, optional matching quotes) but never mutates the process environment — that isolation is the whole point. + + Encoding is ``utf-8-sig`` so a leading UTF-8 BOM (Windows Notepad / + PowerShell ``Set-Content -Encoding UTF8``) does not prefix the first + key as ``\\ufeffNAME`` and make ``get_secret('NAME')`` miss under scope. """ secrets: Dict[str, str] = {} try: - text = env_path.read_text(encoding="utf-8") + text = env_path.read_text(encoding="utf-8-sig") except (FileNotFoundError, OSError, UnicodeDecodeError): return secrets diff --git a/tests/agent/test_secret_scope.py b/tests/agent/test_secret_scope.py index 1ac078d6eb90..29f82a334db6 100644 --- a/tests/agent/test_secret_scope.py +++ b/tests/agent/test_secret_scope.py @@ -116,6 +116,33 @@ class TestEnvFileParsing: + def test_strips_utf8_bom_from_first_key(self, tmp_path): + """Windows editors often save .env as UTF-8 with BOM (EF BB BF). + + Plain utf-8 keeps U+FEFF on the first key name, so get_secret('NAME') + misses under an installed scope. utf-8-sig strips the leading BOM. + """ + env = tmp_path / ".env" + env.write_bytes( + b"\xef\xbb\xbfANTHROPIC_API_KEY=sk-x\nOPENAI_API_KEY=sk-y\n" + ) + out = ss.load_env_file(env) + assert out == { + "ANTHROPIC_API_KEY": "sk-x", + "OPENAI_API_KEY": "sk-y", + } + assert "\ufeffANTHROPIC_API_KEY" not in out + + scope = ss.build_profile_secret_scope(tmp_path) + ss.set_multiplex_active(True) + token = ss.set_secret_scope(scope) + try: + assert ss.get_secret("ANTHROPIC_API_KEY") == "sk-x" + assert ss.get_secret("OPENAI_API_KEY") == "sk-y" + finally: + ss.reset_secret_scope(token) + ss.set_multiplex_active(False) + def test_build_profile_secret_scope(self, tmp_path): (tmp_path / ".env").write_text("ANTHROPIC_API_KEY=sk-profile\n") assert ss.build_profile_secret_scope(tmp_path) == { From 55abf206bff4ed8890da5f35a3e9e119ab9d4c12 Mon Sep 17 00:00:00 2001 From: spfcraze Date: Thu, 30 Jul 2026 19:34:45 -0400 Subject: [PATCH 15/69] fix(agent): unescape quoted .env values in secret_scope.load_env_file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit save_env_value writes values containing " or \ as escaped double-quoted strings, and every other .env reader in the repo (load_env/_parse_env_value, python-dotenv) reverses those escapes. load_env_file — the parser behind build_profile_secret_scope, which wraps every cron job and every multiplexed gateway turn — only stripped the outer quotes, leaving the escapes literal. A credential containing a double quote or backslash (JSON service-account blobs, generated secrets) authenticates fine interactively but 401s under scoped resolution, with no error pointing at the cause. Parse values with the canonical _parse_env_value so all readers agree byte-for-byte. --- agent/secret_scope.py | 21 ++++++++++++++------- tests/agent/test_secret_scope.py | 24 ++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/agent/secret_scope.py b/agent/secret_scope.py index 1e4b30e3a70d..92ae59361cbc 100644 --- a/agent/secret_scope.py +++ b/agent/secret_scope.py @@ -181,9 +181,11 @@ def load_env_file(env_path: Path) -> Dict[str, str]: """Parse a ``.env`` file into a plain dict WITHOUT touching ``os.environ``. Used to load a profile's secrets into an isolated mapping for - ``set_secret_scope``. Mirrors python-dotenv's basic parsing (KEY=VALUE, - ``export`` prefix, ``#`` comments, optional matching quotes) but never - mutates the process environment — that isolation is the whole point. + ``set_secret_scope``. Parses the small KEY=VALUE subset Hermes writes + itself (``export`` prefix, ``#`` comments, matching quotes with the + writer's ``\\"``/``\\\\`` escapes reversed — the same semantics as + ``hermes_cli.config._parse_env_value``) but never mutates the process + environment — that isolation is the whole point. Encoding is ``utf-8-sig`` so a leading UTF-8 BOM (Windows Notepad / PowerShell ``Set-Content -Encoding UTF8``) does not prefix the first @@ -195,6 +197,14 @@ def load_env_file(env_path: Path) -> Dict[str, str]: except (FileNotFoundError, OSError, UnicodeDecodeError): return secrets + # Parse values with the canonical Hermes parser: save_env_value + # escapes " and \ inside double quotes, and every other reader + # (load_env, python-dotenv) reverses those escapes. Stripping only + # the outer quotes here would corrupt credentials containing " + # or \ — they work interactively but fail in scoped (cron / + # multiplex) resolution. + from hermes_cli.config import _parse_env_value + for raw in text.splitlines(): line = raw.strip() if not line or line.startswith("#"): @@ -207,10 +217,7 @@ def load_env_file(env_path: Path) -> Dict[str, str]: key = key.strip() if not key: continue - value = value.strip() - if len(value) >= 2 and value[0] == value[-1] and value[0] in ("'", '"'): - value = value[1:-1] - secrets[key] = value + secrets[key] = _parse_env_value(value) return secrets diff --git a/tests/agent/test_secret_scope.py b/tests/agent/test_secret_scope.py index 29f82a334db6..770921cbde66 100644 --- a/tests/agent/test_secret_scope.py +++ b/tests/agent/test_secret_scope.py @@ -113,6 +113,30 @@ def test_nested_scopes_restore(self): class TestEnvFileParsing: """load_env_file parses without mutating os.environ.""" + def test_load_env_file_unescapes_quoted_values(self, tmp_path): + """Values written by save_env_value must round-trip byte-exactly. + + Regression: load_env_file stripped only the outer quotes, leaving + the writer's \\" and \\\\ escapes literal — credentials containing + '\"' or '\\' worked interactively but were corrupted under scoped + (cron / multiplex) resolution. + """ + from hermes_cli.config import _quote_env_value + + original = 'tok"en\\with spaces' + (tmp_path / ".env").write_text(f"MY_TOKEN={_quote_env_value(original)}\n") + assert ss.load_env_file(tmp_path / ".env") == {"MY_TOKEN": original} + + def test_load_env_file_single_quotes_and_plain_values(self, tmp_path): + (tmp_path / ".env").write_text( + "PLAIN=abc123\nQUOTED='single quoted'\nEMPTY=\n" + ) + assert ss.load_env_file(tmp_path / ".env") == { + "PLAIN": "abc123", + "QUOTED": "single quoted", + "EMPTY": "", + } + From 3fb066c37287a008d6e7b524c083eedb44c6692c Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 2 Aug 2026 00:51:09 -0700 Subject: [PATCH 16/69] fix(agent): dotenv-compatible inline-comment stripping in load_env_file Unquoted values truncate only at a # preceded by whitespace (so KEY=foo#bar stays intact); quoted values scan escape-aware for the matching close quote, keep through it, and drop a trailing '# ...' remainder. Verified empirically against python-dotenv 1.2.2 on a 10-case corpus (full parity). Supersedes the approach in #57718, whose scanner corrupted foo#bar-style values. --- agent/secret_scope.py | 43 +++++++++++++++++++++++++++++-- tests/agent/test_secret_scope.py | 44 ++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 2 deletions(-) diff --git a/agent/secret_scope.py b/agent/secret_scope.py index 92ae59361cbc..1fd87cf14482 100644 --- a/agent/secret_scope.py +++ b/agent/secret_scope.py @@ -23,6 +23,7 @@ from __future__ import annotations import os +import re from contextvars import ContextVar, Token from pathlib import Path from typing import Dict, Mapping, Optional @@ -177,12 +178,50 @@ def get_secret(name: str, default: Optional[str] = None) -> Optional[str]: return val if val is not None else default +def _strip_inline_comment(value: str) -> str: + """Strip a dotenv-style inline comment from a raw ``.env`` value. + + Mirrors python-dotenv (1.2.2) semantics, verified empirically: + + - Quoted values: scan for the matching close quote + (backslash-escape-aware for double quotes, since ``save_env_value`` + writes ``\\"``/``\\\\`` escapes). Everything through the close quote is + kept; a trailing ``# ...`` remainder after it is discarded, so + ``KEY="has # inside" # trailing`` yields ``has # inside``. Non-comment + trailing junk leaves the value untouched (lenient, unlike dotenv's + hard parse error). + - Unquoted values: truncate only at a ``#`` PRECEDED BY WHITESPACE, so + ``KEY=foo#bar`` keeps ``foo#bar`` while ``KEY=value # comment`` keeps + ``value``. A value that *starts* with ``#`` (``KEY=#leading``) is kept. + """ + value = value.strip() + if not value: + return value + quote = value[0] + if quote in ("'", '"'): + i = 1 + while i < len(value): + ch = value[i] + if quote == '"' and ch == "\\": + i += 2 # skip the escaped character + continue + if ch == quote: + remainder = value[i + 1:].lstrip() + if remainder.startswith("#"): + return value[: i + 1] + return value + i += 1 + return value # unterminated quote: leave as-is + return re.split(r"\s+#", value, maxsplit=1)[0].strip() + + def load_env_file(env_path: Path) -> Dict[str, str]: """Parse a ``.env`` file into a plain dict WITHOUT touching ``os.environ``. Used to load a profile's secrets into an isolated mapping for ``set_secret_scope``. Parses the small KEY=VALUE subset Hermes writes - itself (``export`` prefix, ``#`` comments, matching quotes with the + itself (``export`` prefix, ``#`` comments — full-line and + dotenv-compatible inline, matching quotes with the writer's ``\\"``/``\\\\`` escapes reversed — the same semantics as ``hermes_cli.config._parse_env_value``) but never mutates the process environment — that isolation is the whole point. @@ -217,7 +256,7 @@ def load_env_file(env_path: Path) -> Dict[str, str]: key = key.strip() if not key: continue - secrets[key] = _parse_env_value(value) + secrets[key] = _parse_env_value(_strip_inline_comment(value)) return secrets diff --git a/tests/agent/test_secret_scope.py b/tests/agent/test_secret_scope.py index 770921cbde66..ff82eb8b8ac3 100644 --- a/tests/agent/test_secret_scope.py +++ b/tests/agent/test_secret_scope.py @@ -137,6 +137,50 @@ def test_load_env_file_single_quotes_and_plain_values(self, tmp_path): "EMPTY": "", } + def test_inline_comment_stripped_from_unquoted_value(self, tmp_path): + """`KEY=value # comment` → `value` (python-dotenv semantics).""" + (tmp_path / ".env").write_text("KEY=value # comment\nTABBED=foo\t#tabbed\n") + assert ss.load_env_file(tmp_path / ".env") == { + "KEY": "value", + "TABBED": "foo", + } + + def test_hash_without_preceding_whitespace_is_not_a_comment(self, tmp_path): + """`KEY=foo#bar` stays intact — dotenv only strips `#` after whitespace.""" + (tmp_path / ".env").write_text("KEY=foo#bar\nLEAD=#leading\n") + assert ss.load_env_file(tmp_path / ".env") == { + "KEY": "foo#bar", + "LEAD": "#leading", + } + + def test_inline_comment_after_quoted_value(self, tmp_path): + """Quotes strip AND the trailing comment drops; inner `#` survives.""" + (tmp_path / ".env").write_text( + "DQ=\"has # inside\" # trailing\n" + "SQ='single # inside' # trailing\n" + ) + assert ss.load_env_file(tmp_path / ".env") == { + "DQ": "has # inside", + "SQ": "single # inside", + } + + def test_inline_comment_with_escaped_quote_inside_value(self, tmp_path): + r"""Escape-aware close-quote scan: `\"` must not terminate the value.""" + (tmp_path / ".env").write_text( + 'KEY="a \\" quote # x" # trail\n' + ) + assert ss.load_env_file(tmp_path / ".env") == {"KEY": 'a " quote # x'} + + def test_round_trip_writer_value_with_trailing_comment(self, tmp_path): + """A value quoted by the save_env_value writer survives an appended + inline comment byte-exactly.""" + from hermes_cli.config import _quote_env_value + + original = 'we#ird "tok\\en" # not a comment' + quoted = _quote_env_value(original) + (tmp_path / ".env").write_text(f"MY_TOKEN={quoted} # rotated 2026-08\n") + assert ss.load_env_file(tmp_path / ".env") == {"MY_TOKEN": original} + From f7efe3d76663d608578ff78f2f4aee674f41b1d2 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 2 Aug 2026 00:50:20 -0700 Subject: [PATCH 17/69] fix(weixin): Slack-pattern secret scoping for WEIXIN_* credential reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The adapter's __init__ and send_weixin_direct read WEIXIN_ACCOUNT_ID/ TOKEN/BASE_URL/CDN_BASE_URL via bare get_secret, which raises UnscopedSecretError when the DEFAULT profile's adapter constructs or sends unscoped under multiplexing (corrects the direction of #66073 / #68854, which tried to solve this by borrowing os.environ on every read — a cross-profile leak). Add a module-level _wx_secret helper following the established Slack SLACK_APP_TOKEN pattern (#59739) and WhatsApp's _get_wsecret: a SCOPED miss returns the default (the scope is authoritative — no environ borrow), while an UNSCOPED read under multiplex falls back to os.environ, which is the default profile's own value. Regression tests cover both directions: scoped construction reads the scope's value and a scoped miss yields empty (no borrow); unscoped construction falls back to os.environ instead of raising. --- gateway/platforms/weixin.py | 36 ++++++-- tests/gateway/test_weixin_secret_scope.py | 103 ++++++++++++++++++++++ 2 files changed, 130 insertions(+), 9 deletions(-) create mode 100644 tests/gateway/test_weixin_secret_scope.py diff --git a/gateway/platforms/weixin.py b/gateway/platforms/weixin.py index b44ce1ee698f..68a0e0a9e1b1 100644 --- a/gateway/platforms/weixin.py +++ b/gateway/platforms/weixin.py @@ -68,7 +68,25 @@ ) from hermes_constants import get_hermes_home from utils import atomic_json_write -from agent.secret_scope import get_secret +from agent.secret_scope import UnscopedSecretError, get_secret + + +def _wx_secret(name: str, default: Optional[str] = None) -> Optional[str]: + """Scope-aware WEIXIN_* read with the default-profile startup fallback. + + Secondary profiles construct their adapters under + ``_profile_runtime_scope`` — the scope is authoritative and a scoped miss + returns ``default`` (no cross-profile borrow from ``os.environ``). The + DEFAULT profile's adapter constructs and sends *unscoped* under + multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash its Weixin path; there ``os.environ`` is + that profile's own value, so fall back to it. Same pattern as the Slack + ``SLACK_APP_TOKEN`` read (#59739) and WhatsApp's ``_get_wsecret``. + """ + try: + return get_secret(name, default) + except UnscopedSecretError: + return os.getenv(name, default) ILINK_BASE_URL = "https://ilinkai.weixin.qq.com" WEIXIN_CDN_BASE_URL = "https://novac2c.cdn.weixin.qq.com/c2c" @@ -1175,11 +1193,11 @@ def __init__(self, config: PlatformConfig): self._poll_task: Optional[asyncio.Task] = None self._dedup = MessageDeduplicator(ttl_seconds=MESSAGE_DEDUP_TTL_SECONDS) - self._account_id = str(extra.get("account_id") or get_secret("WEIXIN_ACCOUNT_ID", "")).strip() - self._token = str(config.token or extra.get("token") or get_secret("WEIXIN_TOKEN", "")).strip() - self._base_url = str(extra.get("base_url") or get_secret("WEIXIN_BASE_URL", ILINK_BASE_URL)).strip().rstrip("/") + self._account_id = str(extra.get("account_id") or _wx_secret("WEIXIN_ACCOUNT_ID", "")).strip() + self._token = str(config.token or extra.get("token") or _wx_secret("WEIXIN_TOKEN", "")).strip() + self._base_url = str(extra.get("base_url") or _wx_secret("WEIXIN_BASE_URL", ILINK_BASE_URL)).strip().rstrip("/") self._cdn_base_url = str( - extra.get("cdn_base_url") or get_secret("WEIXIN_CDN_BASE_URL", WEIXIN_CDN_BASE_URL) + extra.get("cdn_base_url") or _wx_secret("WEIXIN_CDN_BASE_URL", WEIXIN_CDN_BASE_URL) ).strip().rstrip("/") self._send_chunk_delay_seconds = float( extra.get("send_chunk_delay_seconds") or os.getenv("WEIXIN_SEND_CHUNK_DELAY_SECONDS", "1.5") @@ -2313,10 +2331,10 @@ async def send_weixin_direct( This bypasses the long-poll adapter lifecycle and uses the raw API directly. """ - account_id = str(extra.get("account_id") or get_secret("WEIXIN_ACCOUNT_ID", "")).strip() - base_url = str(extra.get("base_url") or get_secret("WEIXIN_BASE_URL", ILINK_BASE_URL)).strip().rstrip("/") - cdn_base_url = str(extra.get("cdn_base_url") or get_secret("WEIXIN_CDN_BASE_URL", WEIXIN_CDN_BASE_URL)).strip().rstrip("/") - resolved_token = str(token or extra.get("token") or get_secret("WEIXIN_TOKEN", "")).strip() + account_id = str(extra.get("account_id") or _wx_secret("WEIXIN_ACCOUNT_ID", "")).strip() + base_url = str(extra.get("base_url") or _wx_secret("WEIXIN_BASE_URL", ILINK_BASE_URL)).strip().rstrip("/") + cdn_base_url = str(extra.get("cdn_base_url") or _wx_secret("WEIXIN_CDN_BASE_URL", WEIXIN_CDN_BASE_URL)).strip().rstrip("/") + resolved_token = str(token or extra.get("token") or _wx_secret("WEIXIN_TOKEN", "")).strip() if not resolved_token: return {"error": "Weixin token missing. Configure WEIXIN_TOKEN or platforms.weixin.token."} if not account_id: diff --git a/tests/gateway/test_weixin_secret_scope.py b/tests/gateway/test_weixin_secret_scope.py new file mode 100644 index 000000000000..8f5386949f99 --- /dev/null +++ b/tests/gateway/test_weixin_secret_scope.py @@ -0,0 +1,103 @@ +"""Weixin adapter secret-scope regression tests. + +The adapter's WEIXIN_* credential reads must follow the Slack pattern +(#59739): under multiplexing a SCOPED miss is authoritative (no borrow from +``os.environ`` — that would be a cross-profile leak), while an UNSCOPED read +(default-profile startup/send path) falls back to ``os.environ``, which is +that profile's own value, instead of raising ``UnscopedSecretError``. +""" + +import pytest + +from agent import secret_scope +from gateway.config import PlatformConfig +from gateway.platforms.weixin import WeixinAdapter, _wx_secret + + +@pytest.fixture() +def multiplex_on(): + previous = secret_scope.is_multiplex_active() + secret_scope.set_multiplex_active(True) + try: + yield + finally: + secret_scope.set_multiplex_active(previous) + + +class TestWxSecretHelper: + def test_scoped_read_uses_scope_value(self, multiplex_on, monkeypatch): + monkeypatch.setenv("WEIXIN_TOKEN", "default-profile-token") + token = secret_scope.set_secret_scope({"WEIXIN_TOKEN": "scoped-token"}) + try: + assert _wx_secret("WEIXIN_TOKEN") == "scoped-token" + finally: + secret_scope.reset_secret_scope(token) + + def test_scoped_miss_does_not_borrow_environ(self, multiplex_on, monkeypatch): + """A secondary profile without WEIXIN_TOKEN must NOT inherit the + default profile's process-env token.""" + monkeypatch.setenv("WEIXIN_TOKEN", "default-profile-token") + token = secret_scope.set_secret_scope({"OTHER_KEY": "x"}) + try: + assert _wx_secret("WEIXIN_TOKEN", "") == "" + assert _wx_secret("WEIXIN_TOKEN") is None + finally: + secret_scope.reset_secret_scope(token) + + def test_unscoped_read_falls_back_to_environ(self, multiplex_on, monkeypatch): + """The default profile's adapter runs unscoped under multiplexing; + os.environ is its own value — fall back instead of raising.""" + monkeypatch.setenv("WEIXIN_TOKEN", "default-profile-token") + token = secret_scope.set_secret_scope(None) + try: + assert _wx_secret("WEIXIN_TOKEN") == "default-profile-token" + finally: + secret_scope.reset_secret_scope(token) + + +class TestWeixinAdapterConstructionScope: + def test_multiplex_scoped_construction_reads_scope_not_environ( + self, multiplex_on, monkeypatch + ): + monkeypatch.setenv("WEIXIN_ACCOUNT_ID", "env-account") + monkeypatch.setenv("WEIXIN_TOKEN", "env-token") + token = secret_scope.set_secret_scope( + { + "WEIXIN_ACCOUNT_ID": "scoped-account", + "WEIXIN_TOKEN": "scoped-token", + } + ) + try: + adapter = WeixinAdapter(PlatformConfig(enabled=True)) + finally: + secret_scope.reset_secret_scope(token) + assert adapter._account_id == "scoped-account" + assert adapter._token == "scoped-token" + + def test_multiplex_scoped_miss_yields_empty_not_environ_borrow( + self, multiplex_on, monkeypatch + ): + monkeypatch.setenv("WEIXIN_ACCOUNT_ID", "env-account") + monkeypatch.setenv("WEIXIN_TOKEN", "env-token") + token = secret_scope.set_secret_scope({"SOMETHING_ELSE": "x"}) + try: + adapter = WeixinAdapter(PlatformConfig(enabled=True)) + finally: + secret_scope.reset_secret_scope(token) + assert adapter._account_id == "" + assert adapter._token == "" + + def test_multiplex_unscoped_construction_falls_back_to_environ( + self, multiplex_on, monkeypatch + ): + """Regression for the bare get_secret reads: default-profile adapter + construction under multiplexing must not raise UnscopedSecretError.""" + monkeypatch.setenv("WEIXIN_ACCOUNT_ID", "env-account") + monkeypatch.setenv("WEIXIN_TOKEN", "env-token") + token = secret_scope.set_secret_scope(None) + try: + adapter = WeixinAdapter(PlatformConfig(enabled=True)) + finally: + secret_scope.reset_secret_scope(token) + assert adapter._account_id == "env-account" + assert adapter._token == "env-token" From 44640149f75075a58baaa9e6fbf58e18c6f148b3 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 2 Aug 2026 00:52:50 -0700 Subject: [PATCH 18/69] fix(secrets): allowlist API_SERVER listener settings as global deployment env MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #69379 (v2026.7.20 Docker multiplex regression: the scoped runner reload dropped API_SERVER_* set via the container environment, silently losing the api_server platform) by the canonical mechanism — corrects the direction of #69524, which patched gateway/config._getenv to fall through to os.environ on EVERY scoped miss, re-opening the cross-profile borrow for all credentials. API_SERVER_ENABLED / API_SERVER_HOST / API_SERVER_PORT / API_SERVER_CORS_ORIGINS are deployment listener settings (Docker compose environment: block, systemd Environment=), not profile secrets: they join _GLOBAL_ENV_EXACT so get_secret reads them from os.environ regardless of scope. API_SERVER_KEY is deliberately NOT allowlisted — it IS a credential and stays profile-scoped, which keeps tests/gateway/test_config.py's secondary-profile isolation semantics intact (a secondary profile without the key still doesn't bind a listener). Ports #69524's regression test in the corrected form: container-env API_SERVER_* stays visible during the scoped runner reload while the key resolves through the profile scope; plus unit tests locking the allowlist membership and the deliberate exclusion of API_SERVER_KEY. --- agent/secret_scope.py | 8 +++ tests/agent/test_secret_scope.py | 37 ++++++++++++ ...est_64674_multiplex_primary_token_scope.py | 58 +++++++++++++++++++ 3 files changed, 103 insertions(+) diff --git a/agent/secret_scope.py b/agent/secret_scope.py index 1fd87cf14482..919fe3e27bd3 100644 --- a/agent/secret_scope.py +++ b/agent/secret_scope.py @@ -106,6 +106,14 @@ def current_secret_scope() -> Optional[Mapping[str, str]]: "VIRTUAL_ENV", "PYTHONPATH", "SSL_CERT_FILE", # Kanban paths (per-board, not per-profile-secret) "HERMES_KANBAN_DB", "HERMES_KANBAN_WORKSPACES_ROOT", "HERMES_KANBAN_BOARD", + # API-server LISTENER settings — deployment config (Docker compose + # ``environment:`` block, systemd ``Environment=``), not profile secrets. + # The scoped runner reload (#64674) must keep seeing them or container + # deployments silently lose the api_server platform (#69379). NOTE: + # API_SERVER_KEY is deliberately NOT here — it IS a credential and stays + # profile-scoped. + "API_SERVER_ENABLED", "API_SERVER_HOST", "API_SERVER_PORT", + "API_SERVER_CORS_ORIGINS", }) _GLOBAL_ENV_PREFIXES = ( "HERMES_KANBAN_", diff --git a/tests/agent/test_secret_scope.py b/tests/agent/test_secret_scope.py index ff82eb8b8ac3..f0d76fe65435 100644 --- a/tests/agent/test_secret_scope.py +++ b/tests/agent/test_secret_scope.py @@ -250,3 +250,40 @@ def test_build_profile_secret_scope_ignores_other_home_external_secrets( ) assert ss.build_profile_secret_scope(profile) == {} + + +class TestApiServerListenerGlobals: + """API_SERVER listener settings are deployment config (#69379), not + profile secrets: the scoped runner reload must keep seeing container env + (Docker compose ``environment:`` block). API_SERVER_KEY IS a credential + and stays profile-scoped.""" + + LISTENER_VARS = ( + "API_SERVER_ENABLED", + "API_SERVER_HOST", + "API_SERVER_PORT", + "API_SERVER_CORS_ORIGINS", + ) + + def test_listener_vars_read_environ_even_when_scoped_multiplex(self, monkeypatch): + for name in self.LISTENER_VARS: + monkeypatch.setenv(name, f"container-{name.lower()}") + ss.set_multiplex_active(True) + token = ss.set_secret_scope({"TELEGRAM_BOT_TOKEN": "scoped"}) + try: + for name in self.LISTENER_VARS: + assert ss.get_secret(name) == f"container-{name.lower()}" + finally: + ss.reset_secret_scope(token) + + def test_api_server_key_stays_profile_scoped(self, monkeypatch): + monkeypatch.setenv("API_SERVER_KEY", "default-profile-key-0123456789abcdef") + ss.set_multiplex_active(True) + token = ss.set_secret_scope({"OTHER": "x"}) + try: + # A scoped miss must NOT borrow the (potentially cross-profile) + # environ value: API_SERVER_KEY is a credential. + assert ss.get_secret("API_SERVER_KEY") is None + finally: + ss.reset_secret_scope(token) + assert not ss._is_global_env("API_SERVER_KEY") diff --git a/tests/gateway/test_64674_multiplex_primary_token_scope.py b/tests/gateway/test_64674_multiplex_primary_token_scope.py index 1ce5f91abdf2..44398aec19f6 100644 --- a/tests/gateway/test_64674_multiplex_primary_token_scope.py +++ b/tests/gateway/test_64674_multiplex_primary_token_scope.py @@ -49,6 +49,64 @@ def test_unscoped_when_multiplex_off(self, tmp_path, monkeypatch): cfg = run_mod.load_gateway_config_for_runner() assert cfg.multiplex_profiles is False + def test_scoped_reload_still_sees_container_api_server_env(self, tmp_path, monkeypatch): + """#69379 — container-env API_SERVER_* visible during the scoped reload. + + Docker/systemd deployments enable the api_server platform via the + process environment (compose ``environment:`` block), not the profile + ``.env``. The multiplex runner reload happens inside the default + profile's secret scope; the listener settings are on the global + allowlist (deployment config, not profile secrets) so they must stay + visible there — while API_SERVER_KEY (a credential) still resolves + through the profile scope. + """ + from agent import secret_scope as ss + from gateway import run as run_mod + + home = tmp_path / "home" + home.mkdir() + # Credentials belong in the profile .env; listener settings do not. + (home / ".env").write_text( + "TELEGRAM_BOT_TOKEN=default-profile-token-123\n" + "API_SERVER_KEY=profile-scoped-key-0123456789abcdef\n", + encoding="utf-8", + ) + (home / "config.yaml").write_text( + "gateway:\n multiplex_profiles: true\n", encoding="utf-8" + ) + monkeypatch.setenv("HERMES_HOME", str(home)) + # Listener settings live ONLY in os.environ — the Docker compose case. + monkeypatch.setenv("API_SERVER_ENABLED", "true") + monkeypatch.setenv("API_SERVER_HOST", "0.0.0.0") + monkeypatch.setenv("API_SERVER_PORT", "8642") + monkeypatch.delenv("API_SERVER_KEY", raising=False) + monkeypatch.delenv("TELEGRAM_BOT_TOKEN", raising=False) + monkeypatch.setattr(run_mod, "get_hermes_home", lambda: home) + monkeypatch.setattr(run_mod, "_hermes_home", home) + # Model the real multiplexed gateway: run.py flips the runtime flag + # before the runner reload, making any installed scope authoritative. + ss.set_multiplex_active(True) + + cfg = run_mod.load_gateway_config_for_runner() + + assert cfg.multiplex_profiles is True + # Telegram token from the profile scope (.env) + tg = cfg.platforms.get(Platform.TELEGRAM) + assert tg is not None + assert tg.token == "default-profile-token-123" + # api_server present: key from the profile scope, listener settings + # from the container environment via the global allowlist. + api = cfg.platforms.get(Platform.API_SERVER) + assert api is not None, ( + "api_server should be enabled from container env even inside " + "the scoped runner reload (#69379)" + ) + assert api.enabled is True + assert api.extra.get("key") == "profile-scoped-key-0123456789abcdef" + assert api.extra.get("host") == "0.0.0.0" + assert api.extra.get("port") == 8642 + + class TestPlatformHasBotCredential: def test_telegram_empty_token_false(self): From 062d44bba7973aa911d086be2be495c8db980744 Mon Sep 17 00:00:00 2001 From: Ray Date: Sun, 2 Aug 2026 00:53:16 -0700 Subject: [PATCH 19/69] =?UTF-8?q?fix(xai):=20fail=20closed=20in=20xai=5Fht?= =?UTF-8?q?tp.get=5Fenv=5Fvalue=20=E2=80=94=20honor=20get=5Fsecret's=20ver?= =?UTF-8?q?dict?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Salvaged from #56982 (@rayjun): the live piece of the PR. The hermes_cli/config.py get_env_value scope-honoring change and its test_env_load_cache.py tests were already merged via ed1170cd8b (#76462) and are dropped here. tools/xai_http.py::get_env_value wrapped the scope-aware hermes_cli.config.get_env_value in except Exception + a raw os.environ fallback — swallowing UnscopedSecretError and borrowing the process env, so a multiplexed xAI credential read could silently pick up another profile's XAI_API_KEY. Narrow the except to ImportError (the only legitimate degraded case) so get_secret's verdict propagates: an unscoped multiplexed read fails closed, and a scoped miss returns the default instead of the foreign environ value. Co-authored-by: rayjun --- tests/tools/test_xai_http_credentials.py | 57 ++++++++++++++++++++++++ tools/xai_http.py | 10 ++--- 2 files changed, 61 insertions(+), 6 deletions(-) create mode 100644 tests/tools/test_xai_http_credentials.py diff --git a/tests/tools/test_xai_http_credentials.py b/tests/tools/test_xai_http_credentials.py new file mode 100644 index 000000000000..4217823f8c83 --- /dev/null +++ b/tests/tools/test_xai_http_credentials.py @@ -0,0 +1,57 @@ +import pytest + + +def _set_xai_oauth_unavailable(monkeypatch): + from hermes_cli import auth + + monkeypatch.setattr(auth, "resolve_xai_oauth_runtime_credentials", lambda **_: {}) + + +def test_xai_credentials_fail_closed_without_profile_scope(tmp_path, monkeypatch): + from agent import secret_scope + from hermes_cli.config import invalidate_env_cache + from tools.xai_http import resolve_xai_http_credentials + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("XAI_API_KEY", "foreign-xai-key") + monkeypatch.setenv("XAI_BASE_URL", "https://foreign.example/v1") + _set_xai_oauth_unavailable(monkeypatch) + invalidate_env_cache() + previous_multiplex = secret_scope.is_multiplex_active() + token = secret_scope.set_secret_scope(None) + secret_scope.set_multiplex_active(True) + try: + with pytest.raises(secret_scope.UnscopedSecretError): + resolve_xai_http_credentials(force_refresh=True) + finally: + secret_scope.reset_secret_scope(token) + secret_scope.set_multiplex_active(previous_multiplex) + invalidate_env_cache() + + +def test_xai_credentials_do_not_fall_back_to_environ_when_scope_has_no_key( + tmp_path, monkeypatch +): + from agent import secret_scope + from hermes_cli.config import invalidate_env_cache + from tools.xai_http import resolve_xai_http_credentials + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("XAI_API_KEY", "foreign-xai-key") + monkeypatch.setenv("XAI_BASE_URL", "https://foreign.example/v1") + _set_xai_oauth_unavailable(monkeypatch) + invalidate_env_cache() + previous_multiplex = secret_scope.is_multiplex_active() + token = secret_scope.set_secret_scope({}) + secret_scope.set_multiplex_active(True) + try: + credentials = resolve_xai_http_credentials(force_refresh=True) + assert credentials == { + "provider": "xai", + "api_key": "", + "base_url": "https://api.x.ai/v1", + } + finally: + secret_scope.reset_secret_scope(token) + secret_scope.set_multiplex_active(previous_multiplex) + invalidate_env_cache() diff --git a/tools/xai_http.py b/tools/xai_http.py index 8ef0b856302d..1e4f72a7fb0a 100644 --- a/tools/xai_http.py +++ b/tools/xai_http.py @@ -79,13 +79,11 @@ def get_env_value(name: str, default=None): """ try: from hermes_cli.config import get_env_value as _hermes_get_env_value + except ImportError: + return os.environ.get(name, default) - value = _hermes_get_env_value(name) - if value is not None: - return value - except Exception: - pass - return os.environ.get(name, default) + value = _hermes_get_env_value(name) + return value if value is not None else default def hermes_xai_user_agent() -> str: From d7404f197c6af7ec5a586865de4e028f2fa55389 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 2 Aug 2026 00:53:55 -0700 Subject: [PATCH 20/69] =?UTF-8?q?fix(secrets):=20Slack-pattern=20scoped=20?= =?UTF-8?q?credential=20reads=20=E2=80=94=20IRC,=20LINE,=20Teams,=20Matter?= =?UTF-8?q?most?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Route IRC_SERVER_PASSWORD/IRC_NICKSERV_PASSWORD, LINE_CHANNEL_ACCESS_TOKEN/ LINE_CHANNEL_SECRET, TEAMS_GRAPH_ACCESS_TOKEN/TEAMS_CLIENT_SECRET and MATTERMOST_TOKEN reads at __init__/availability/standalone-send time through a module-level _get_scoped_secret helper (get_secret, UnscopedSecretError -> os.getenv fallback), mirroring whatsapp_common._get_wsecret / Slack #59739. Scoped miss returns the default — no cross-profile environ borrow. --- plugins/platforms/irc/adapter.py | 40 ++++++++++++++++++++----- plugins/platforms/line/adapter.py | 40 ++++++++++++++++++++----- plugins/platforms/mattermost/adapter.py | 30 +++++++++++++++++-- plugins/platforms/teams/adapter.py | 34 +++++++++++++++++---- 4 files changed, 120 insertions(+), 24 deletions(-) diff --git a/plugins/platforms/irc/adapter.py b/plugins/platforms/irc/adapter.py index e78798adbe67..030300b7970b 100644 --- a/plugins/platforms/irc/adapter.py +++ b/plugins/platforms/irc/adapter.py @@ -35,6 +35,30 @@ import time from typing import Any, Dict, List, Optional +from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError +from agent.secret_scope import get_secret as _scoped_get_secret + + +def _get_scoped_secret(name, default=None): + """Scope-aware credential read with the default-profile startup fallback. + + Secondary profiles construct their adapters under a profile secret + scope -- the scope is authoritative and a scoped miss returns ``default`` + (no cross-profile borrow from ``os.environ``, which may hold another + profile's value). The DEFAULT profile's adapter constructs and sends + *unscoped* under multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash this path; there ``os.environ`` is that + profile's own value, so fall back to it. Same pattern as the Slack + ``SLACK_APP_TOKEN`` read (#59739) and + ``gateway/platforms/whatsapp_common.py::_get_wsecret``. + """ + try: + val = _scoped_get_secret(name, default) + except _UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- @@ -118,8 +142,8 @@ def __init__(self, config, **kwargs): if os.getenv("IRC_USE_TLS") else extra.get("use_tls", True) ) - self.server_password = os.getenv("IRC_SERVER_PASSWORD") or extra.get("server_password", "") - self.nickserv_password = os.getenv("IRC_NICKSERV_PASSWORD") or extra.get("nickserv_password", "") + self.server_password = _get_scoped_secret("IRC_SERVER_PASSWORD") or extra.get("server_password", "") + self.nickserv_password = _get_scoped_secret("IRC_NICKSERV_PASSWORD") or extra.get("nickserv_password", "") # Auth self.allowed_users: list = extra.get("allowed_users", []) @@ -685,10 +709,10 @@ def _env_enablement() -> dict | None: seed["use_tls"] = use_tls in {"1", "true", "yes"} # Passwords live in PlatformConfig.extra as well for back-compat with # existing config.yaml users; env-reads at construct time still win. - if os.getenv("IRC_SERVER_PASSWORD"): - seed["server_password"] = os.getenv("IRC_SERVER_PASSWORD") - if os.getenv("IRC_NICKSERV_PASSWORD"): - seed["nickserv_password"] = os.getenv("IRC_NICKSERV_PASSWORD") + if _get_scoped_secret("IRC_SERVER_PASSWORD"): + seed["server_password"] = _get_scoped_secret("IRC_SERVER_PASSWORD") + if _get_scoped_secret("IRC_NICKSERV_PASSWORD"): + seed["nickserv_password"] = _get_scoped_secret("IRC_NICKSERV_PASSWORD") # Optional home-channel (usually the same as IRC_CHANNEL, but can be a # dedicated reports channel). Defaults to IRC_CHANNEL so cron jobs # with ``deliver=irc`` have a sensible target without extra config. @@ -762,8 +786,8 @@ async def _standalone_send( else: use_tls = bool(extra.get("use_tls", True)) - server_password = os.getenv("IRC_SERVER_PASSWORD") or extra.get("server_password", "") - nickserv_password = os.getenv("IRC_NICKSERV_PASSWORD") or extra.get("nickserv_password", "") + server_password = _get_scoped_secret("IRC_SERVER_PASSWORD") or extra.get("server_password", "") + nickserv_password = _get_scoped_secret("IRC_NICKSERV_PASSWORD") or extra.get("nickserv_password", "") # Reject control characters in chat_id to block IRC command injection. raw_target = chat_id or channel diff --git a/plugins/platforms/line/adapter.py b/plugins/platforms/line/adapter.py index cf14782693d1..b3c9c6463ef1 100644 --- a/plugins/platforms/line/adapter.py +++ b/plugins/platforms/line/adapter.py @@ -80,6 +80,30 @@ from typing import Any, Dict, List, Optional, Set, Tuple from urllib.parse import quote as _urlquote +from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError +from agent.secret_scope import get_secret as _scoped_get_secret + + +def _get_scoped_secret(name, default=None): + """Scope-aware credential read with the default-profile startup fallback. + + Secondary profiles construct their adapters under a profile secret + scope -- the scope is authoritative and a scoped miss returns ``default`` + (no cross-profile borrow from ``os.environ``, which may hold another + profile's value). The DEFAULT profile's adapter constructs and sends + *unscoped* under multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash this path; there ``os.environ`` is that + profile's own value, so fall back to it. Same pattern as the Slack + ``SLACK_APP_TOKEN`` read (#59739) and + ``gateway/platforms/whatsapp_common.py::_get_wsecret``. + """ + try: + val = _scoped_get_secret(name, default) + except _UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- @@ -678,11 +702,11 @@ def __init__(self, config, **kwargs): # Credentials self.channel_access_token = ( - os.getenv("LINE_CHANNEL_ACCESS_TOKEN") + _get_scoped_secret("LINE_CHANNEL_ACCESS_TOKEN") or extra.get("channel_access_token", "") ) self.channel_secret = ( - os.getenv("LINE_CHANNEL_SECRET") + _get_scoped_secret("LINE_CHANNEL_SECRET") or extra.get("channel_secret", "") ) @@ -1561,9 +1585,9 @@ def _is_relative_to(child: Path, parent: Path) -> bool: def check_requirements() -> bool: """Plugin gate: require credentials AND aiohttp at runtime.""" - if not os.getenv("LINE_CHANNEL_ACCESS_TOKEN"): + if not _get_scoped_secret("LINE_CHANNEL_ACCESS_TOKEN"): return False - if not os.getenv("LINE_CHANNEL_SECRET"): + if not _get_scoped_secret("LINE_CHANNEL_SECRET"): return False try: import aiohttp # noqa: F401 @@ -1575,10 +1599,10 @@ def check_requirements() -> bool: def validate_config(config) -> bool: extra = getattr(config, "extra", {}) or {} has_token = bool( - os.getenv("LINE_CHANNEL_ACCESS_TOKEN") or extra.get("channel_access_token") + _get_scoped_secret("LINE_CHANNEL_ACCESS_TOKEN") or extra.get("channel_access_token") ) has_secret = bool( - os.getenv("LINE_CHANNEL_SECRET") or extra.get("channel_secret") + _get_scoped_secret("LINE_CHANNEL_SECRET") or extra.get("channel_secret") ) return has_token and has_secret @@ -1595,7 +1619,7 @@ def _env_enablement() -> Optional[Dict[str, Any]]: in ``.env`` without a ``platforms.line`` block in ``config.yaml``. Mirrors the IRC plugin's pattern. """ - if not (os.getenv("LINE_CHANNEL_ACCESS_TOKEN") and os.getenv("LINE_CHANNEL_SECRET")): + if not (_get_scoped_secret("LINE_CHANNEL_ACCESS_TOKEN") and _get_scoped_secret("LINE_CHANNEL_SECRET")): return None seeded: Dict[str, Any] = {} if os.getenv("LINE_PORT"): @@ -1635,7 +1659,7 @@ async def _standalone_send( """ extra = getattr(pconfig, "extra", {}) or {} token = ( - os.getenv("LINE_CHANNEL_ACCESS_TOKEN") + _get_scoped_secret("LINE_CHANNEL_ACCESS_TOKEN") or extra.get("channel_access_token", "") ) if not token or not chat_id: diff --git a/plugins/platforms/mattermost/adapter.py b/plugins/platforms/mattermost/adapter.py index 63e7cf266f8e..c1239b37def5 100644 --- a/plugins/platforms/mattermost/adapter.py +++ b/plugins/platforms/mattermost/adapter.py @@ -30,6 +30,30 @@ SendResult, ) +from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError +from agent.secret_scope import get_secret as _scoped_get_secret + + +def _get_scoped_secret(name, default=None): + """Scope-aware credential read with the default-profile startup fallback. + + Secondary profiles construct their adapters under a profile secret + scope -- the scope is authoritative and a scoped miss returns ``default`` + (no cross-profile borrow from ``os.environ``, which may hold another + profile's value). The DEFAULT profile's adapter constructs and sends + *unscoped* under multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash this path; there ``os.environ`` is that + profile's own value, so fall back to it. Same pattern as the Slack + ``SLACK_APP_TOKEN`` read (#59739) and + ``gateway/platforms/whatsapp_common.py::_get_wsecret``. + """ + try: + val = _scoped_get_secret(name, default) + except _UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + logger = logging.getLogger(__name__) # Mattermost post size limit (server default is 16383, but 4000 is the @@ -75,7 +99,7 @@ def check_mattermost_requirements() -> bool: def validate_mattermost_config(config: PlatformConfig) -> bool: """Return True when Mattermost has enough config to connect.""" extra = getattr(config, "extra", {}) or {} - token = (getattr(config, "token", None) or os.getenv("MATTERMOST_TOKEN", "")).strip() + token = (getattr(config, "token", None) or _get_scoped_secret("MATTERMOST_TOKEN", "")).strip() url = (extra.get("url", "") or os.getenv("MATTERMOST_URL", "")).strip() if not token: logger.debug("Mattermost: MATTERMOST_TOKEN not set") @@ -98,7 +122,7 @@ def __init__(self, config: PlatformConfig): config.extra.get("url", "") or os.getenv("MATTERMOST_URL", "") ).rstrip("/") - self._token: str = config.token or os.getenv("MATTERMOST_TOKEN", "") + self._token: str = config.token or _get_scoped_secret("MATTERMOST_TOKEN", "") self._bot_user_id: str = "" self._bot_username: str = "" @@ -1022,7 +1046,7 @@ async def _standalone_send( (getattr(pconfig, "extra", {}) or {}).get("url") or os.getenv("MATTERMOST_URL", "") ).rstrip("/") - token = (getattr(pconfig, "token", None) or os.getenv("MATTERMOST_TOKEN", "")).strip() + token = (getattr(pconfig, "token", None) or _get_scoped_secret("MATTERMOST_TOKEN", "")).strip() if not base_url or not token: return { "error": ( diff --git a/plugins/platforms/teams/adapter.py b/plugins/platforms/teams/adapter.py index d5e7035ae996..9ccbb8aaa51f 100644 --- a/plugins/platforms/teams/adapter.py +++ b/plugins/platforms/teams/adapter.py @@ -91,6 +91,30 @@ cache_media_bytes, ) +from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError +from agent.secret_scope import get_secret as _scoped_get_secret + + +def _get_scoped_secret(name, default=None): + """Scope-aware credential read with the default-profile startup fallback. + + Secondary profiles construct their adapters under a profile secret + scope -- the scope is authoritative and a scoped miss returns ``default`` + (no cross-profile borrow from ``os.environ``, which may hold another + profile's value). The DEFAULT profile's adapter constructs and sends + *unscoped* under multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash this path; there ``os.environ`` is that + profile's own value, so fall back to it. Same pattern as the Slack + ``SLACK_APP_TOKEN`` read (#59739) and + ``gateway/platforms/whatsapp_common.py::_get_wsecret``. + """ + try: + val = _scoped_get_secret(name, default) + except _UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + logger = logging.getLogger(__name__) _DEFAULT_PORT = 3978 @@ -200,7 +224,7 @@ def _resolve_delivery_config(self, config: dict[str, Any] | None) -> dict[str, A env_defaults = { "delivery_mode": os.getenv("TEAMS_DELIVERY_MODE", ""), "incoming_webhook_url": os.getenv("TEAMS_INCOMING_WEBHOOK_URL", ""), - "access_token": os.getenv("TEAMS_GRAPH_ACCESS_TOKEN", ""), + "access_token": _get_scoped_secret("TEAMS_GRAPH_ACCESS_TOKEN", ""), "team_id": os.getenv("TEAMS_TEAM_ID", ""), "channel_id": os.getenv("TEAMS_CHANNEL_ID", ""), "chat_id": os.getenv("TEAMS_CHAT_ID", ""), @@ -401,7 +425,7 @@ def validate_config(config) -> bool: """Return True when the config has the minimum required credentials.""" extra = getattr(config, "extra", {}) or {} client_id = os.getenv("TEAMS_CLIENT_ID") or extra.get("client_id", "") - client_secret = os.getenv("TEAMS_CLIENT_SECRET") or extra.get("client_secret", "") + client_secret = _get_scoped_secret("TEAMS_CLIENT_SECRET") or extra.get("client_secret", "") tenant_id = os.getenv("TEAMS_TENANT_ID") or extra.get("tenant_id", "") return bool(client_id and client_secret and tenant_id) @@ -423,7 +447,7 @@ def _env_enablement() -> dict | None: ``HomeChannel`` dataclass on the ``PlatformConfig`` via the core hook. """ client_id = os.getenv("TEAMS_CLIENT_ID", "").strip() - client_secret = os.getenv("TEAMS_CLIENT_SECRET", "").strip() + client_secret = _get_scoped_secret("TEAMS_CLIENT_SECRET", "").strip() tenant_id = os.getenv("TEAMS_TENANT_ID", "").strip() if not (client_id and client_secret and tenant_id): return None @@ -528,7 +552,7 @@ async def _standalone_send( """ extra = getattr(pconfig, "extra", {}) or {} client_id = os.getenv("TEAMS_CLIENT_ID") or extra.get("client_id", "") - client_secret = os.getenv("TEAMS_CLIENT_SECRET") or extra.get("client_secret", "") + client_secret = _get_scoped_secret("TEAMS_CLIENT_SECRET") or extra.get("client_secret", "") tenant_id = os.getenv("TEAMS_TENANT_ID") or extra.get("tenant_id", "") if not (client_id and client_secret and tenant_id): return {"error": "Teams standalone send: TEAMS_CLIENT_ID, TEAMS_CLIENT_SECRET, and TEAMS_TENANT_ID are all required"} @@ -728,7 +752,7 @@ def __init__(self, config: PlatformConfig): super().__init__(config, Platform("teams")) extra = config.extra or {} self._client_id = extra.get("client_id") or os.getenv("TEAMS_CLIENT_ID", "") - self._client_secret = extra.get("client_secret") or os.getenv("TEAMS_CLIENT_SECRET", "") + self._client_secret = extra.get("client_secret") or _get_scoped_secret("TEAMS_CLIENT_SECRET", "") self._tenant_id = extra.get("tenant_id") or os.getenv("TEAMS_TENANT_ID", "") self._port = _coerce_port( extra.get("port") or os.getenv("TEAMS_PORT", str(_DEFAULT_PORT)) From 6333180c9ad4a4385966316fcfef46369bb7dff1 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 2 Aug 2026 00:53:55 -0700 Subject: [PATCH 21/69] =?UTF-8?q?fix(secrets):=20Slack-pattern=20scoped=20?= =?UTF-8?q?credential=20reads=20=E2=80=94=20ntfy,=20Home=20Assistant,=20SM?= =?UTF-8?q?S,=20DingTalk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NTFY_TOKEN, HASS_TOKEN, TWILIO_ACCOUNT_SID/TWILIO_AUTH_TOKEN and DINGTALK_CLIENT_SECRET now read through _get_scoped_secret. Also replaces the SMS adapter's bare os.environ["TWILIO_AUTH_TOKEN"]/["TWILIO_ACCOUNT_SID"] __init__ reads (KeyError-prone) with helper reads defaulting to "". --- plugins/platforms/dingtalk/adapter.py | 30 +++++++++++++++++-- plugins/platforms/homeassistant/adapter.py | 30 +++++++++++++++++-- plugins/platforms/ntfy/adapter.py | 30 +++++++++++++++++-- plugins/platforms/sms/adapter.py | 34 ++++++++++++++++++---- 4 files changed, 110 insertions(+), 14 deletions(-) diff --git a/plugins/platforms/dingtalk/adapter.py b/plugins/platforms/dingtalk/adapter.py index a564c08e690a..9ea8330854ce 100644 --- a/plugins/platforms/dingtalk/adapter.py +++ b/plugins/platforms/dingtalk/adapter.py @@ -103,6 +103,30 @@ SendResult, ) +from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError +from agent.secret_scope import get_secret as _scoped_get_secret + + +def _get_scoped_secret(name, default=None): + """Scope-aware credential read with the default-profile startup fallback. + + Secondary profiles construct their adapters under a profile secret + scope -- the scope is authoritative and a scoped miss returns ``default`` + (no cross-profile borrow from ``os.environ``, which may hold another + profile's value). The DEFAULT profile's adapter constructs and sends + *unscoped* under multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash this path; there ``os.environ`` is that + profile's own value, so fall back to it. Same pattern as the Slack + ``SLACK_APP_TOKEN`` read (#59739) and + ``gateway/platforms/whatsapp_common.py::_get_wsecret``. + """ + try: + val = _scoped_get_secret(name, default) + except _UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + logger = logging.getLogger(__name__) MAX_MESSAGE_LENGTH = 20000 @@ -166,7 +190,7 @@ def check_dingtalk_requirements() -> bool: httpx = _httpx DINGTALK_STREAM_AVAILABLE = True HTTPX_AVAILABLE = True - if not os.getenv("DINGTALK_CLIENT_ID") or not os.getenv("DINGTALK_CLIENT_SECRET"): + if not os.getenv("DINGTALK_CLIENT_ID") or not _get_scoped_secret("DINGTALK_CLIENT_SECRET"): return False return True @@ -213,7 +237,7 @@ def __init__(self, config: PlatformConfig): self._client_id: str = extra.get("client_id") or os.getenv( "DINGTALK_CLIENT_ID", "" ) - self._client_secret: str = extra.get("client_secret") or os.getenv( + self._client_secret: str = extra.get("client_secret") or _get_scoped_secret( "DINGTALK_CLIENT_SECRET", "" ) @@ -1842,7 +1866,7 @@ def _is_connected(config) -> bool: extra = getattr(config, "extra", {}) or {} return bool( (extra.get("client_id") or os.getenv("DINGTALK_CLIENT_ID")) - and (extra.get("client_secret") or os.getenv("DINGTALK_CLIENT_SECRET")) + and (extra.get("client_secret") or _get_scoped_secret("DINGTALK_CLIENT_SECRET")) ) diff --git a/plugins/platforms/homeassistant/adapter.py b/plugins/platforms/homeassistant/adapter.py index 8e042133eee8..6564460d47dd 100644 --- a/plugins/platforms/homeassistant/adapter.py +++ b/plugins/platforms/homeassistant/adapter.py @@ -36,6 +36,30 @@ SendResult, ) +from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError +from agent.secret_scope import get_secret as _scoped_get_secret + + +def _get_scoped_secret(name, default=None): + """Scope-aware credential read with the default-profile startup fallback. + + Secondary profiles construct their adapters under a profile secret + scope -- the scope is authoritative and a scoped miss returns ``default`` + (no cross-profile borrow from ``os.environ``, which may hold another + profile's value). The DEFAULT profile's adapter constructs and sends + *unscoped* under multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash this path; there ``os.environ`` is that + profile's own value, so fall back to it. Same pattern as the Slack + ``SLACK_APP_TOKEN`` read (#59739) and + ``gateway/platforms/whatsapp_common.py::_get_wsecret``. + """ + try: + val = _scoped_get_secret(name, default) + except _UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + logger = logging.getLogger(__name__) @@ -46,7 +70,7 @@ def check_ha_requirements() -> bool: def validate_ha_config(config: PlatformConfig) -> bool: """Return True when Home Assistant has enough credential config to connect.""" - token = (getattr(config, "token", None) or os.getenv("HASS_TOKEN", "")).strip() + token = (getattr(config, "token", None) or _get_scoped_secret("HASS_TOKEN", "")).strip() return bool(token) @@ -76,7 +100,7 @@ def __init__(self, config: PlatformConfig): # Configuration from extra extra = config.extra or {} - token = config.token or os.getenv("HASS_TOKEN", "") + token = config.token or _get_scoped_secret("HASS_TOKEN", "") url = extra.get("url") or os.getenv("HASS_URL", "http://homeassistant.local:8123") self._hass_url: str = url.rstrip("/") self._hass_token: str = token @@ -488,7 +512,7 @@ async def _standalone_send( extra = getattr(pconfig, "extra", {}) or {} hass_url = (extra.get("url") or os.getenv("HASS_URL", "")).rstrip("/") - token = (getattr(pconfig, "token", None) or os.getenv("HASS_TOKEN", "")).strip() + token = (getattr(pconfig, "token", None) or _get_scoped_secret("HASS_TOKEN", "")).strip() if not hass_url or not token: return { "error": ( diff --git a/plugins/platforms/ntfy/adapter.py b/plugins/platforms/ntfy/adapter.py index 88741aa62f5c..935610c32046 100644 --- a/plugins/platforms/ntfy/adapter.py +++ b/plugins/platforms/ntfy/adapter.py @@ -68,6 +68,30 @@ SendResult, ) +from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError +from agent.secret_scope import get_secret as _scoped_get_secret + + +def _get_scoped_secret(name, default=None): + """Scope-aware credential read with the default-profile startup fallback. + + Secondary profiles construct their adapters under a profile secret + scope -- the scope is authoritative and a scoped miss returns ``default`` + (no cross-profile borrow from ``os.environ``, which may hold another + profile's value). The DEFAULT profile's adapter constructs and sends + *unscoped* under multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash this path; there ``os.environ`` is that + profile's own value, so fall back to it. Same pattern as the Slack + ``SLACK_APP_TOKEN`` read (#59739) and + ``gateway/platforms/whatsapp_common.py::_get_wsecret``. + """ + try: + val = _scoped_get_secret(name, default) + except _UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + logger = logging.getLogger(__name__) @@ -173,7 +197,7 @@ def __init__(self, config: PlatformConfig): or os.getenv("NTFY_PUBLISH_TOPIC", "") or self._topic ) - self._token: str = extra.get("token") or os.getenv("NTFY_TOKEN", "") + self._token: str = extra.get("token") or _get_scoped_secret("NTFY_TOKEN", "") self._stream_task: Optional[asyncio.Task] = None self._http_client: Optional["httpx.AsyncClient"] = None @@ -472,7 +496,7 @@ def _env_enablement() -> dict | None: publish_topic = os.getenv("NTFY_PUBLISH_TOPIC", "").strip() if publish_topic: seed["publish_topic"] = publish_topic - token = os.getenv("NTFY_TOKEN", "").strip() + token = _get_scoped_secret("NTFY_TOKEN", "").strip() if token: seed["token"] = token markdown = os.getenv("NTFY_MARKDOWN", "").strip().lower() @@ -526,7 +550,7 @@ async def _standalone_send( if not publish_topic: return {"error": "ntfy standalone send: NTFY_TOPIC not configured"} - token = extra.get("token") or os.getenv("NTFY_TOKEN", "") + token = extra.get("token") or _get_scoped_secret("NTFY_TOKEN", "") markdown_env = os.getenv("NTFY_MARKDOWN", "").strip().lower() markdown_enabled = bool(extra.get("markdown")) or markdown_env in ("1", "true", "yes") diff --git a/plugins/platforms/sms/adapter.py b/plugins/platforms/sms/adapter.py index 3d794da592ae..0c081242d969 100644 --- a/plugins/platforms/sms/adapter.py +++ b/plugins/platforms/sms/adapter.py @@ -36,6 +36,30 @@ ) from gateway.platforms.helpers import redact_phone, strip_markdown +from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError +from agent.secret_scope import get_secret as _scoped_get_secret + + +def _get_scoped_secret(name, default=None): + """Scope-aware credential read with the default-profile startup fallback. + + Secondary profiles construct their adapters under a profile secret + scope -- the scope is authoritative and a scoped miss returns ``default`` + (no cross-profile borrow from ``os.environ``, which may hold another + profile's value). The DEFAULT profile's adapter constructs and sends + *unscoped* under multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash this path; there ``os.environ`` is that + profile's own value, so fall back to it. Same pattern as the Slack + ``SLACK_APP_TOKEN`` read (#59739) and + ``gateway/platforms/whatsapp_common.py::_get_wsecret``. + """ + try: + val = _scoped_get_secret(name, default) + except _UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + logger = logging.getLogger(__name__) TWILIO_API_BASE = "https://api.twilio.com/2010-04-01/Accounts" @@ -51,7 +75,7 @@ def check_sms_requirements() -> bool: import aiohttp # noqa: F401 except ImportError: return False - return bool(os.getenv("TWILIO_ACCOUNT_SID") and os.getenv("TWILIO_AUTH_TOKEN")) + return bool(_get_scoped_secret("TWILIO_ACCOUNT_SID") and _get_scoped_secret("TWILIO_AUTH_TOKEN")) class SmsAdapter(BasePlatformAdapter): @@ -66,8 +90,8 @@ class SmsAdapter(BasePlatformAdapter): def __init__(self, config: PlatformConfig): super().__init__(config, Platform.SMS) - self._account_sid: str = os.environ["TWILIO_ACCOUNT_SID"] - self._auth_token: str = os.environ["TWILIO_AUTH_TOKEN"] + self._account_sid: str = _get_scoped_secret("TWILIO_ACCOUNT_SID", "") + self._auth_token: str = _get_scoped_secret("TWILIO_AUTH_TOKEN", "") self._from_number: str = os.getenv("TWILIO_PHONE_NUMBER", "") self._webhook_port: int = int( os.getenv("SMS_WEBHOOK_PORT", str(DEFAULT_WEBHOOK_PORT)) @@ -435,14 +459,14 @@ async def _standalone_send( ): """Out-of-process SMS delivery via the Twilio REST API. Implements the standalone_sender_fn contract; replaces the legacy _send_sms helper.""" - auth_token = getattr(pconfig, "api_key", None) or os.getenv("TWILIO_AUTH_TOKEN", "") + auth_token = getattr(pconfig, "api_key", None) or _get_scoped_secret("TWILIO_AUTH_TOKEN", "") try: import aiohttp except ImportError: return {"error": "aiohttp not installed. Run: pip install aiohttp"} import base64 - account_sid = os.getenv("TWILIO_ACCOUNT_SID", "") + account_sid = _get_scoped_secret("TWILIO_ACCOUNT_SID", "") from_number = os.getenv("TWILIO_PHONE_NUMBER", "") if not account_sid or not auth_token or not from_number: return {"error": "SMS not configured (TWILIO_ACCOUNT_SID, TWILIO_AUTH_TOKEN, TWILIO_PHONE_NUMBER required)"} From f4b268b7852694abc8207cbf5dacebcc1cff9942 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 2 Aug 2026 00:53:55 -0700 Subject: [PATCH 22/69] =?UTF-8?q?fix(secrets):=20Slack-pattern=20scoped=20?= =?UTF-8?q?credential=20reads=20=E2=80=94=20Feishu,=20WeCom,=20Photon,=20B?= =?UTF-8?q?uzz?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FEISHU_APP_SECRET/FEISHU_ENCRYPT_KEY/FEISHU_VERIFICATION_TOKEN, WECOM_SECRET, PHOTON_PROJECT_SECRET/PHOTON_SIDECAR_TOKEN (adapter + auth.load_project_credentials) and BUZZ_PRIVATE_KEY now read through _get_scoped_secret. --- plugins/platforms/buzz/adapter.py | 26 ++++++++++++++++++++++- plugins/platforms/feishu/adapter.py | 30 ++++++++++++++++++++++++--- plugins/platforms/photon/adapter.py | 32 +++++++++++++++++++++++++---- plugins/platforms/photon/auth.py | 26 ++++++++++++++++++++++- plugins/platforms/wecom/adapter.py | 26 ++++++++++++++++++++++- 5 files changed, 130 insertions(+), 10 deletions(-) diff --git a/plugins/platforms/buzz/adapter.py b/plugins/platforms/buzz/adapter.py index b07830ae9b6e..8b77ae334120 100644 --- a/plugins/platforms/buzz/adapter.py +++ b/plugins/platforms/buzz/adapter.py @@ -49,6 +49,30 @@ from typing import Any, Dict, List, Optional, Tuple from urllib.parse import urlsplit, urlunsplit +from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError +from agent.secret_scope import get_secret as _scoped_get_secret + + +def _get_scoped_secret(name, default=None): + """Scope-aware credential read with the default-profile startup fallback. + + Secondary profiles construct their adapters under a profile secret + scope -- the scope is authoritative and a scoped miss returns ``default`` + (no cross-profile borrow from ``os.environ``, which may hold another + profile's value). The DEFAULT profile's adapter constructs and sends + *unscoped* under multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash this path; there ``os.environ`` is that + profile's own value, so fall back to it. Same pattern as the Slack + ``SLACK_APP_TOKEN`` read (#59739) and + ``gateway/platforms/whatsapp_common.py::_get_wsecret``. + """ + try: + val = _scoped_get_secret(name, default) + except _UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + logger = logging.getLogger(__name__) from gateway.platforms.base import ( @@ -226,7 +250,7 @@ def _resolve_private_key(extra: Optional[dict] = None) -> str: NEVER log the return value. """ - key = os.getenv("BUZZ_PRIVATE_KEY", "").strip() + key = _get_scoped_secret("BUZZ_PRIVATE_KEY", "").strip() if key: return key configured = os.getenv("BUZZ_CREDENTIALS_FILE", "").strip() or (extra or {}).get("credentials_file", "") diff --git a/plugins/platforms/feishu/adapter.py b/plugins/platforms/feishu/adapter.py index d86dc597a816..c21bb42e9011 100644 --- a/plugins/platforms/feishu/adapter.py +++ b/plugins/platforms/feishu/adapter.py @@ -145,6 +145,30 @@ from hermes_constants import get_hermes_home from utils import atomic_json_write, env_float, env_int +from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError +from agent.secret_scope import get_secret as _scoped_get_secret + + +def _get_scoped_secret(name, default=None): + """Scope-aware credential read with the default-profile startup fallback. + + Secondary profiles construct their adapters under a profile secret + scope -- the scope is authoritative and a scoped miss returns ``default`` + (no cross-profile borrow from ``os.environ``, which may hold another + profile's value). The DEFAULT profile's adapter constructs and sends + *unscoped* under multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash this path; there ``os.environ`` is that + profile's own value, so fall back to it. Same pattern as the Slack + ``SLACK_APP_TOKEN`` read (#59739) and + ``gateway/platforms/whatsapp_common.py::_get_wsecret``. + """ + try: + val = _scoped_get_secret(name, default) + except _UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- @@ -1553,14 +1577,14 @@ def _load_settings(extra: Dict[str, Any]) -> FeishuAdapterSettings: return FeishuAdapterSettings( app_id=str(extra.get("app_id") or os.getenv("FEISHU_APP_ID", "")).strip(), - app_secret=str(extra.get("app_secret") or os.getenv("FEISHU_APP_SECRET", "")).strip(), + app_secret=str(extra.get("app_secret") or _get_scoped_secret("FEISHU_APP_SECRET", "")).strip(), domain_name=str(extra.get("domain") or os.getenv("FEISHU_DOMAIN", "feishu")).strip().lower(), connection_mode=str( extra.get("connection_mode") or os.getenv("FEISHU_CONNECTION_MODE", "websocket") ).strip().lower(), - encrypt_key=str(extra.get("encrypt_key") or os.getenv("FEISHU_ENCRYPT_KEY", "")).strip(), + encrypt_key=str(extra.get("encrypt_key") or _get_scoped_secret("FEISHU_ENCRYPT_KEY", "")).strip(), verification_token=str( - extra.get("verification_token") or os.getenv("FEISHU_VERIFICATION_TOKEN", "") + extra.get("verification_token") or _get_scoped_secret("FEISHU_VERIFICATION_TOKEN", "") ).strip(), group_policy=os.getenv("FEISHU_GROUP_POLICY", "allowlist").strip().lower(), allowed_group_users=frozenset( diff --git a/plugins/platforms/photon/adapter.py b/plugins/platforms/photon/adapter.py index abebe3669f98..d7f89c4a6925 100644 --- a/plugins/platforms/photon/adapter.py +++ b/plugins/platforms/photon/adapter.py @@ -67,6 +67,30 @@ from .auth import load_project_credentials +from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError +from agent.secret_scope import get_secret as _scoped_get_secret + + +def _get_scoped_secret(name, default=None): + """Scope-aware credential read with the default-profile startup fallback. + + Secondary profiles construct their adapters under a profile secret + scope -- the scope is authoritative and a scoped miss returns ``default`` + (no cross-profile borrow from ``os.environ``, which may hold another + profile's value). The DEFAULT profile's adapter constructs and sends + *unscoped* under multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash this path; there ``os.environ`` is that + profile's own value, so fall back to it. Same pattern as the Slack + ``SLACK_APP_TOKEN`` read (#59739) and + ``gateway/platforms/whatsapp_common.py::_get_wsecret``. + """ + try: + val = _scoped_get_secret(name, default) + except _UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- @@ -511,7 +535,7 @@ def _reinstall_sidecar_deps() -> None: def validate_config(cfg: PlatformConfig) -> bool: extra = cfg.extra or {} project_id = extra.get("project_id") or os.getenv("PHOTON_PROJECT_ID") - project_secret = extra.get("project_secret") or os.getenv("PHOTON_PROJECT_SECRET") + project_secret = extra.get("project_secret") or _get_scoped_secret("PHOTON_PROJECT_SECRET") if not project_id or not project_secret: # Fall back to auth.json stored_id, stored_sec = load_project_credentials() @@ -694,7 +718,7 @@ def __init__(self, config: PlatformConfig): or "" ) self._project_secret: str = ( - os.getenv("PHOTON_PROJECT_SECRET") + _get_scoped_secret("PHOTON_PROJECT_SECRET") or extra.get("project_secret") or stored_sec or "" @@ -707,7 +731,7 @@ def __init__(self, config: PlatformConfig): ) self._sidecar_bind = _DEFAULT_SIDECAR_BIND self._sidecar_token = ( - os.getenv("PHOTON_SIDECAR_TOKEN") or secrets.token_hex(16) + _get_scoped_secret("PHOTON_SIDECAR_TOKEN") or secrets.token_hex(16) ) self._autostart_sidecar = str( os.getenv("PHOTON_SIDECAR_AUTOSTART", "true") @@ -2730,7 +2754,7 @@ async def _standalone_send( (pconfig.extra or {}).get("sidecar_port") or os.getenv("PHOTON_SIDECAR_PORT"), _DEFAULT_SIDECAR_PORT, ) - token = os.getenv("PHOTON_SIDECAR_TOKEN") + token = _get_scoped_secret("PHOTON_SIDECAR_TOKEN") if not token: # Fall back to the runtime record the gateway persists once its # sidecar passes /healthz (issue #69960) — the token only exists in diff --git a/plugins/platforms/photon/auth.py b/plugins/platforms/photon/auth.py index b8b356a16b8c..34b573a2d810 100644 --- a/plugins/platforms/photon/auth.py +++ b/plugins/platforms/photon/auth.py @@ -53,6 +53,30 @@ except ImportError: # pragma: no cover - httpx is a hermes dependency httpx = None # type: ignore[assignment] +from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError +from agent.secret_scope import get_secret as _scoped_get_secret + + +def _get_scoped_secret(name, default=None): + """Scope-aware credential read with the default-profile startup fallback. + + Secondary profiles construct their adapters under a profile secret + scope -- the scope is authoritative and a scoped miss returns ``default`` + (no cross-profile borrow from ``os.environ``, which may hold another + profile's value). The DEFAULT profile's adapter constructs and sends + *unscoped* under multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash this path; there ``os.environ`` is that + profile's own value, so fall back to it. Same pattern as the Slack + ``SLACK_APP_TOKEN`` read (#59739) and + ``gateway/platforms/whatsapp_common.py::_get_wsecret``. + """ + try: + val = _scoped_get_secret(name, default) + except _UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + logger = logging.getLogger(__name__) @@ -231,7 +255,7 @@ def load_project_credentials() -> Tuple[Optional[str], Optional[str]]: is the unified project id (dashboard id == spectrumProjectId). """ env_id = os.getenv("PHOTON_PROJECT_ID") - env_sec = os.getenv("PHOTON_PROJECT_SECRET") + env_sec = _get_scoped_secret("PHOTON_PROJECT_SECRET") if env_id and env_sec: return env_id, env_sec auth = _load_auth() diff --git a/plugins/platforms/wecom/adapter.py b/plugins/platforms/wecom/adapter.py index 2f8e6d66fc31..715445bd236b 100644 --- a/plugins/platforms/wecom/adapter.py +++ b/plugins/platforms/wecom/adapter.py @@ -70,6 +70,30 @@ ) from utils import env_float +from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError +from agent.secret_scope import get_secret as _scoped_get_secret + + +def _get_scoped_secret(name, default=None): + """Scope-aware credential read with the default-profile startup fallback. + + Secondary profiles construct their adapters under a profile secret + scope -- the scope is authoritative and a scoped miss returns ``default`` + (no cross-profile borrow from ``os.environ``, which may hold another + profile's value). The DEFAULT profile's adapter constructs and sends + *unscoped* under multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash this path; there ``os.environ`` is that + profile's own value, so fall back to it. Same pattern as the Slack + ``SLACK_APP_TOKEN`` read (#59739) and + ``gateway/platforms/whatsapp_common.py::_get_wsecret``. + """ + try: + val = _scoped_get_secret(name, default) + except _UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + logger = logging.getLogger(__name__) DEFAULT_WS_URL = "wss://openws.work.weixin.qq.com" @@ -154,7 +178,7 @@ def __init__(self, config: PlatformConfig): extra = config.extra or {} self._bot_id = str(extra.get("bot_id") or os.getenv("WECOM_BOT_ID", "")).strip() - self._secret = str(extra.get("secret") or os.getenv("WECOM_SECRET", "")).strip() + self._secret = str(extra.get("secret") or _get_scoped_secret("WECOM_SECRET", "")).strip() self._ws_url = str( extra.get("websocket_url") or extra.get("websocketUrl") From 7d4c8f9a541318d4b81cabdb483c36017932af0e Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 2 Aug 2026 00:53:55 -0700 Subject: [PATCH 23/69] fix(secrets): scoped BLUEBUBBLES_PASSWORD + API_SERVER_KEY init read; add parametrized regression tests api_server's __init__ API_SERVER_KEY read now matches the scoped _expected_api_key path. tests/gateway/test_adapter_startup_secret_scope.py asserts, for every migrated module: helper exists, scoped read wins, scoped miss returns default (no environ borrow), unscoped-under-multiplex falls back to environ without raising, and legacy single-profile reads still work. --- gateway/platforms/api_server.py | 26 +++- gateway/platforms/bluebubbles.py | 26 +++- .../test_adapter_startup_secret_scope.py | 125 ++++++++++++++++++ 3 files changed, 175 insertions(+), 2 deletions(-) create mode 100644 tests/gateway/test_adapter_startup_secret_scope.py diff --git a/gateway/platforms/api_server.py b/gateway/platforms/api_server.py index 8f346a4599e4..bd69e203f242 100644 --- a/gateway/platforms/api_server.py +++ b/gateway/platforms/api_server.py @@ -95,6 +95,30 @@ def _approval_event_choices(*, smart_denied: bool, allow_permanent: bool) -> lis from agent.interrupt_compat import request_hard_interrupt from gateway.readiness import collect_runtime_readiness +from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError +from agent.secret_scope import get_secret as _scoped_get_secret + + +def _get_scoped_secret(name, default=None): + """Scope-aware credential read with the default-profile startup fallback. + + Secondary profiles construct their adapters under a profile secret + scope -- the scope is authoritative and a scoped miss returns ``default`` + (no cross-profile borrow from ``os.environ``, which may hold another + profile's value). The DEFAULT profile's adapter constructs and sends + *unscoped* under multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash this path; there ``os.environ`` is that + profile's own value, so fall back to it. Same pattern as the Slack + ``SLACK_APP_TOKEN`` read (#59739) and + ``gateway/platforms/whatsapp_common.py::_get_wsecret``. + """ + try: + val = _scoped_get_secret(name, default) + except _UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + logger = logging.getLogger(__name__) @@ -1301,7 +1325,7 @@ def __init__(self, config: PlatformConfig): if raw_port is None: raw_port = os.getenv("API_SERVER_PORT", str(DEFAULT_PORT)) self._port: int = _coerce_port(raw_port, DEFAULT_PORT) - self._api_key: str = extra.get("key", os.getenv("API_SERVER_KEY", "")) + self._api_key: str = extra.get("key", _get_scoped_secret("API_SERVER_KEY", "")) self._cors_origins: tuple[str, ...] = self._parse_cors_origins( extra.get("cors_origins", os.getenv("API_SERVER_CORS_ORIGINS", "")), ) diff --git a/gateway/platforms/bluebubbles.py b/gateway/platforms/bluebubbles.py index 1cb8c040cd90..ecc6baf45e01 100644 --- a/gateway/platforms/bluebubbles.py +++ b/gateway/platforms/bluebubbles.py @@ -56,6 +56,30 @@ "audio/aac": ".m4a", # preserves historical bluebubbles mapping (shared table says .aac) } +from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError +from agent.secret_scope import get_secret as _scoped_get_secret + + +def _get_scoped_secret(name, default=None): + """Scope-aware credential read with the default-profile startup fallback. + + Secondary profiles construct their adapters under a profile secret + scope -- the scope is authoritative and a scoped miss returns ``default`` + (no cross-profile borrow from ``os.environ``, which may hold another + profile's value). The DEFAULT profile's adapter constructs and sends + *unscoped* under multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash this path; there ``os.environ`` is that + profile's own value, so fall back to it. Same pattern as the Slack + ``SLACK_APP_TOKEN`` read (#59739) and + ``gateway/platforms/whatsapp_common.py::_get_wsecret``. + """ + try: + val = _scoped_get_secret(name, default) + except _UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- @@ -148,7 +172,7 @@ def __init__(self, config: PlatformConfig): self.server_url = _normalize_server_url( extra.get("server_url") or os.getenv("BLUEBUBBLES_SERVER_URL", "") ) - self.password = extra.get("password") or os.getenv("BLUEBUBBLES_PASSWORD", "") + self.password = extra.get("password") or _get_scoped_secret("BLUEBUBBLES_PASSWORD", "") self.webhook_host = ( extra.get("webhook_host") or os.getenv("BLUEBUBBLES_WEBHOOK_HOST", DEFAULT_WEBHOOK_HOST) diff --git a/tests/gateway/test_adapter_startup_secret_scope.py b/tests/gateway/test_adapter_startup_secret_scope.py new file mode 100644 index 000000000000..8e8260436c3d --- /dev/null +++ b/tests/gateway/test_adapter_startup_secret_scope.py @@ -0,0 +1,125 @@ +"""Regression tests — Slack-pattern scoped credential reads at adapter startup. + +Class-closure follow-up to the profile secret-scope cluster (#76462, Slack +pattern #59739, WhatsApp ``_get_wsecret``): the 13 platform adapters below +read credentials at ``__init__`` / availability-check / standalone-send time +with bare ``os.getenv`` (SMS even used ``os.environ[...]``, which KeyErrors). +Under ``gateway.multiplex_profiles`` those reads leak the default profile's +credential into secondary profiles' adapters. + +Each migrated module now carries a module-level ``_get_scoped_secret`` helper +mirroring ``gateway/platforms/whatsapp_common.py::_get_wsecret``: + +- scope installed → scope is authoritative; scoped miss returns the default + (NO borrow from ``os.environ``) +- unscoped under multiplex (the default profile's own startup loop) → + fall back to ``os.getenv`` without raising ``UnscopedSecretError`` + +The helpers are imported and exercised directly rather than constructing every +adapter — construction pulls heavy platform dependencies (slack-bolt-style SDK +imports, webhook servers, sidecars) that the test environment doesn't need. +""" + +import importlib + +import pytest + +from agent import secret_scope as ss + +# (module path, representative credential env var owned by that adapter) +MIGRATED_ADAPTER_MODULES = [ + ("plugins.platforms.irc.adapter", "IRC_SERVER_PASSWORD"), + ("plugins.platforms.line.adapter", "LINE_CHANNEL_ACCESS_TOKEN"), + ("plugins.platforms.teams.adapter", "TEAMS_CLIENT_SECRET"), + ("plugins.platforms.mattermost.adapter", "MATTERMOST_TOKEN"), + ("plugins.platforms.ntfy.adapter", "NTFY_TOKEN"), + ("plugins.platforms.homeassistant.adapter", "HASS_TOKEN"), + ("plugins.platforms.sms.adapter", "TWILIO_AUTH_TOKEN"), + ("plugins.platforms.dingtalk.adapter", "DINGTALK_CLIENT_SECRET"), + ("plugins.platforms.feishu.adapter", "FEISHU_APP_SECRET"), + ("plugins.platforms.wecom.adapter", "WECOM_SECRET"), + ("plugins.platforms.photon.adapter", "PHOTON_PROJECT_SECRET"), + ("plugins.platforms.photon.auth", "PHOTON_PROJECT_SECRET"), + ("plugins.platforms.buzz.adapter", "BUZZ_PRIVATE_KEY"), + ("gateway.platforms.bluebubbles", "BLUEBUBBLES_PASSWORD"), + ("gateway.platforms.api_server", "API_SERVER_KEY"), +] + +MODULE_IDS = [m for m, _ in MIGRATED_ADAPTER_MODULES] + + +@pytest.fixture(autouse=True) +def _reset_multiplex(): + """Multiplex mode off before and after every test.""" + ss.set_multiplex_active(False) + yield + ss.set_multiplex_active(False) + + +def _helper(module_name): + mod = importlib.import_module(module_name) + helper = getattr(mod, "_get_scoped_secret", None) + assert helper is not None, ( + f"{module_name} must define the module-level _get_scoped_secret helper " + "(Slack pattern #59739 / whatsapp_common._get_wsecret)" + ) + return helper + + +@pytest.mark.parametrize(("module_name", "var"), MIGRATED_ADAPTER_MODULES, ids=MODULE_IDS) +def test_helper_exists(module_name, var): + _helper(module_name) + + +@pytest.mark.parametrize(("module_name", "var"), MIGRATED_ADAPTER_MODULES, ids=MODULE_IDS) +def test_scoped_read_wins_over_environ(module_name, var, monkeypatch): + """Scope installed under multiplex: the profile's value wins, not environ's.""" + helper = _helper(module_name) + monkeypatch.setenv(var, "default-profile-value") + ss.set_multiplex_active(True) + tok = ss.set_secret_scope({var: "secondary-profile-value"}) + try: + assert helper(var) == "secondary-profile-value" + finally: + ss.reset_secret_scope(tok) + + +@pytest.mark.parametrize(("module_name", "var"), MIGRATED_ADAPTER_MODULES, ids=MODULE_IDS) +def test_scoped_miss_returns_default_no_environ_borrow(module_name, var, monkeypatch): + """Scope installed but key absent: return the default — never borrow the + default profile's os.environ value into a secondary profile.""" + helper = _helper(module_name) + monkeypatch.setenv(var, "default-profile-value") + ss.set_multiplex_active(True) + tok = ss.set_secret_scope({"SOME_OTHER_KEY": "x"}) + try: + assert helper(var) is None + assert helper(var, "") == "" + assert helper(var, "sentinel") == "sentinel" + finally: + ss.reset_secret_scope(tok) + + +@pytest.mark.parametrize(("module_name", "var"), MIGRATED_ADAPTER_MODULES, ids=MODULE_IDS) +def test_unscoped_under_multiplex_falls_back_to_environ(module_name, var, monkeypatch): + """The DEFAULT profile constructs its adapters unscoped under multiplexing. + A bare get_secret would raise UnscopedSecretError and crash startup; the + helper must fall back to os.environ (that profile's own value) instead.""" + helper = _helper(module_name) + monkeypatch.setenv(var, "default-profile-own-value") + ss.set_multiplex_active(True) + assert ss.current_secret_scope() is None + assert helper(var) == "default-profile-own-value" + + monkeypatch.delenv(var, raising=False) + assert helper(var, "fallback-default") == "fallback-default" + + +@pytest.mark.parametrize(("module_name", "var"), MIGRATED_ADAPTER_MODULES, ids=MODULE_IDS) +def test_single_profile_legacy_environ_read(module_name, var, monkeypatch): + """Multiplex off, no scope: legacy os.environ read keeps working.""" + helper = _helper(module_name) + monkeypatch.setenv(var, "legacy-env-value") + assert helper(var) == "legacy-env-value" + monkeypatch.delenv(var, raising=False) + assert helper(var, "d") == "d" From 224e59df5293d7095064cc0320ef5bca7aa862ae Mon Sep 17 00:00:00 2001 From: Da7-Tech <286182457+Da7-Tech@users.noreply.github.com> Date: Tue, 7 Jul 2026 21:31:18 +0300 Subject: [PATCH 24/69] fix(qqbot): resolve credentials under the active profile secret scope MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The QQ adapter read QQ_APP_ID, QQ_CLIENT_SECRET, the QQ_STT_* backend config and the QQ_ALLOW_ALL_USERS policy flag through raw os.getenv, bypassing the active profile secret scope. In multiplex mode a secondary profile whose secret lives in its own .env (installed as an isolated scope, not into os.environ) would silently fall back to the default/primary profile's value — the same cross-profile collision fixed for the WeChat/weixin adapter in #59662. Route these reads through a scope-aware resolver that reads the profile scope when one is installed (secondary profiles and per-turn inbound) and falls back to os.environ otherwise. The fallback is deliberate: the primary/active profile is constructed without a scope and owns os.environ, so a bare get_secret would raise UnscopedSecretError and break its startup. Mirrors gateway.config._getenv. Adds regression tests including active-profile-no-scope construction (the fail-closed case), plus scope-wins-over-environ, two-profile isolation, single-profile fallback, explicit-config precedence and STT key scoping. --- gateway/platforms/qqbot/adapter.py | 39 +++++-- .../test_qqbot_credential_isolation.py | 102 ++++++++++++++++++ 2 files changed, 135 insertions(+), 6 deletions(-) create mode 100644 tests/gateway/test_qqbot_credential_isolation.py diff --git a/gateway/platforms/qqbot/adapter.py b/gateway/platforms/qqbot/adapter.py index 8f5d7fececf9..d540692c095f 100644 --- a/gateway/platforms/qqbot/adapter.py +++ b/gateway/platforms/qqbot/adapter.py @@ -147,6 +147,31 @@ def _coerce_list(value: Any) -> List[str]: return _coerce_list_impl(value) +def _resolve_qq_secret(name: str, default: str = "") -> str: + """Resolve a per-profile ``QQ_*`` setting honoring the active secret scope. + + When a profile secret scope is installed — every secondary multiplex + profile is constructed and handled inside ``_profile_runtime_scope`` + (``gateway/run.py``), as is each per-turn inbound message — read from it so + profiles never see each other's ``os.environ`` values. This is the + cross-profile credential collision fixed for the WeChat adapter in #59662. + + The primary/active profile is constructed without a scope and legitimately + owns ``os.environ``, so fall back to it there instead of failing closed: a + bare ``get_secret`` would raise ``UnscopedSecretError`` on the active + profile's ``__init__`` and break its startup. Same pattern as the Slack + ``SLACK_APP_TOKEN`` read (#59739) and + ``gateway.platforms.whatsapp_common._get_wsecret``. + """ + from agent.secret_scope import UnscopedSecretError, get_secret + + try: + val = get_secret(name, default) + except UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + # --------------------------------------------------------------------------- # QQAdapter # --------------------------------------------------------------------------- @@ -202,9 +227,11 @@ def __init__(self, config: PlatformConfig): super().__init__(config, Platform.QQBOT) extra = config.extra or {} - self._app_id = str(extra.get("app_id") or os.getenv("QQ_APP_ID", "")).strip() + self._app_id = str( + extra.get("app_id") or _resolve_qq_secret("QQ_APP_ID", "") + ).strip() self._client_secret = str( - extra.get("client_secret") or os.getenv("QQ_CLIENT_SECRET", "") + extra.get("client_secret") or _resolve_qq_secret("QQ_CLIENT_SECRET", "") ).strip() self._markdown_support = bool(extra.get("markdown_support", True)) @@ -2202,13 +2229,13 @@ def _resolve_stt_config(self) -> Optional[Dict[str, str]]: } # 2. QQ-specific env vars (set by `hermes setup gateway` / `hermes gateway`) - qq_stt_key = os.getenv("QQ_STT_API_KEY", "") + qq_stt_key = _resolve_qq_secret("QQ_STT_API_KEY", "") if qq_stt_key: - base_url = os.getenv( + base_url = _resolve_qq_secret( "QQ_STT_BASE_URL", "https://open.bigmodel.cn/api/coding/paas/v4", ) - model = os.getenv("QQ_STT_MODEL", "glm-asr") + model = _resolve_qq_secret("QQ_STT_MODEL", "glm-asr") return { "base_url": base_url.rstrip("/"), "api_key": qq_stt_key, @@ -3170,7 +3197,7 @@ def _strip_at_mention(content: str) -> str: def _open_dm_opted_in(self) -> bool: if os.getenv("GATEWAY_ALLOW_ALL_USERS", "").lower() in {"true", "1", "yes"}: return True - return os.getenv("QQ_ALLOW_ALL_USERS", "").lower() in {"true", "1", "yes"} + return _resolve_qq_secret("QQ_ALLOW_ALL_USERS", "").lower() in {"true", "1", "yes"} def _is_dm_allowed(self, user_id: str) -> bool: if self._dm_policy == "disabled": diff --git a/tests/gateway/test_qqbot_credential_isolation.py b/tests/gateway/test_qqbot_credential_isolation.py new file mode 100644 index 000000000000..693ea59cd598 --- /dev/null +++ b/tests/gateway/test_qqbot_credential_isolation.py @@ -0,0 +1,102 @@ +"""Credential isolation for the QQ (qqbot) gateway adapter. + +Covers the multiplex credential-collision class (same class as the +WeChat/weixin adapter tracked in #59662): the QQ adapter resolves its ``QQ_*`` +settings through the active profile secret scope rather than raw ``os.getenv``, +so a secondary profile whose secret lives in its own ``.env`` (installed as an +isolated scope, not into ``os.environ``) does not fall back to the +default/primary profile's value. + +Also guards the primary/active profile: it is constructed without a scope and +legitimately owns ``os.environ``, so the resolver must fall back to +``os.environ`` there (not fail closed) even when multiplexing is active — +otherwise the active profile's adapter would raise ``UnscopedSecretError`` on +construction and fail to start. +""" +import pytest + +from agent import secret_scope as ss +from gateway.config import PlatformConfig +from gateway.platforms.qqbot.adapter import QQAdapter + + +@pytest.fixture(autouse=True) +def _reset_multiplex(): + ss.set_multiplex_active(False) + yield + ss.set_multiplex_active(False) + + +def _make_adapter(extra=None): + return QQAdapter(PlatformConfig(enabled=True, extra=extra or {})) + + +class TestQQCredentialScope: + def test_credentials_read_scope_not_environ(self, monkeypatch): + # os.environ holds another profile's values; the scoped values must win. + monkeypatch.setenv("QQ_APP_ID", "global-app") + monkeypatch.setenv("QQ_CLIENT_SECRET", "global-secret") + ss.set_multiplex_active(True) + tok = ss.set_secret_scope( + {"QQ_APP_ID": "profileA-app", "QQ_CLIENT_SECRET": "profileA-secret"} + ) + try: + adapter = _make_adapter() + finally: + ss.reset_secret_scope(tok) + assert adapter._app_id == "profileA-app" + assert adapter._client_secret == "profileA-secret" + + def test_two_profiles_isolated(self): + ss.set_multiplex_active(True) + tok_a = ss.set_secret_scope({"QQ_CLIENT_SECRET": "secret-A"}) + try: + a = _make_adapter() + finally: + ss.reset_secret_scope(tok_a) + tok_b = ss.set_secret_scope({"QQ_CLIENT_SECRET": "secret-B"}) + try: + b = _make_adapter() + finally: + ss.reset_secret_scope(tok_b) + assert a._client_secret == "secret-A" + assert b._client_secret == "secret-B" + + def test_single_profile_still_reads_environ(self, monkeypatch): + # No scope + multiplex inactive (default single-profile deployment): + # legacy os.environ behavior is preserved — no regression. + monkeypatch.setenv("QQ_CLIENT_SECRET", "legacy-secret") + adapter = _make_adapter() + assert adapter._client_secret == "legacy-secret" + + def test_active_profile_no_scope_reads_environ_without_raising(self, monkeypatch): + # The primary/active profile is built with NO scope while multiplexing + # is active. A bare get_secret() would fail closed (UnscopedSecretError) + # and break its startup; the resolver must fall back to os.environ. + monkeypatch.setenv("QQ_APP_ID", "primary-app") + monkeypatch.setenv("QQ_CLIENT_SECRET", "primary-secret") + ss.set_multiplex_active(True) + assert ss.current_secret_scope() is None # no scope installed + adapter = _make_adapter() # must not raise + assert adapter._app_id == "primary-app" + assert adapter._client_secret == "primary-secret" + + def test_explicit_config_extra_takes_precedence(self, monkeypatch): + # An explicit value in config.extra still wins over env/scope. + monkeypatch.setenv("QQ_CLIENT_SECRET", "env-secret") + adapter = _make_adapter(extra={"client_secret": "explicit"}) + assert adapter._client_secret == "explicit" + + +class TestQQSttConfigScope: + def test_stt_api_key_reads_scope(self, monkeypatch): + monkeypatch.setenv("QQ_STT_API_KEY", "global-stt-key") + ss.set_multiplex_active(True) + tok = ss.set_secret_scope({"QQ_STT_API_KEY": "profileA-stt-key"}) + try: + adapter = _make_adapter() + stt = adapter._resolve_stt_config() + finally: + ss.reset_secret_scope(tok) + assert stt is not None + assert stt["api_key"] == "profileA-stt-key" From e8c5cb57104229cc1bac8490d9089c1036fcd5f0 Mon Sep 17 00:00:00 2001 From: Da7-Tech <286182457+Da7-Tech@users.noreply.github.com> Date: Thu, 16 Jul 2026 06:13:43 +0300 Subject: [PATCH 25/69] fix(qqbot): scope the authz, startup-validation, and direct-send QQ reads Review follow-up: the adapter-level resolver alone left three paths reading per-profile QQ_* values from raw os.getenv, so a secondary multiplex profile's scoped opt-in or credentials were ignored (or the primary's environ values leaked in): - gateway/authz_mixin.py: route the per-platform allow-all flag and the per-platform/group allowlist + allow-bots reads through the scope-aware gateway.config._getenv. Deployment-global GATEWAY_* reads intentionally stay on os.getenv. This makes the same fix effective for every own-policy platform, not just QQ; unscoped behavior is byte-identical to os.getenv. - gateway/run.py (_own_policy_open_startup_violation): resolve the per-platform dm/group policy and allow-all opt-in via _getenv; the secondary-profile caller already runs inside _profile_runtime_scope. - tools/send_message_tool.py (_send_qqbot): the QQ_APP_ID / QQ_CLIENT_SECRET fallbacks now honor the active profile scope. Tests: tests/gateway/test_qqbot_scope_paths.py covers all three paths end-to-end (scope wins, no environ inheritance for non-opted profiles, single-profile environ fallback unchanged); the STT suite now asserts QQ_STT_BASE_URL and QQ_STT_MODEL scoping alongside the API key. All five scoped-behavior tests fail on the previous commit and pass here. --- gateway/run.py | 7 +- .../test_qqbot_credential_isolation.py | 23 ++ tests/gateway/test_qqbot_scope_paths.py | 227 ++++++++++++++++++ tools/send_message_tool.py | 9 +- 4 files changed, 261 insertions(+), 5 deletions(-) create mode 100644 tests/gateway/test_qqbot_scope_paths.py diff --git a/gateway/run.py b/gateway/run.py index adb867f80d2d..6c97d02260ba 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -2205,6 +2205,7 @@ def _platform_has_bot_credential(platform: "Platform", platform_config: "Platfor _BUILTIN_PLATFORM_VALUES, GatewayConfig, PlatformConfig, + _getenv, load_gateway_config, ) from gateway.session import ( @@ -2294,11 +2295,11 @@ def _own_policy_open_startup_violation(config) -> Optional[str]: extra = getattr(platform_config, "extra", None) or {} dm_policy = str( extra.get("dm_policy") - or (os.getenv(dm_env, "pairing") if dm_env else "pairing") + or (_getenv(dm_env, "pairing") if dm_env else "pairing") ).strip().lower() group_policy = str( extra.get("group_policy") - or (os.getenv(group_env, "pairing") if group_env else "pairing") + or (_getenv(group_env, "pairing") if group_env else "pairing") ).strip().lower() if dm_policy != "open" and group_policy != "open": continue @@ -2307,7 +2308,7 @@ def _own_policy_open_startup_violation(config) -> Optional[str]: ).lower() in {"true", "1", "yes"} platform_opted_in = gateway_allow_all or ( allow_all_env - and os.getenv(allow_all_env, "").lower() in {"true", "1", "yes"} + and _getenv(allow_all_env, "").lower() in {"true", "1", "yes"} ) if platform_opted_in: continue diff --git a/tests/gateway/test_qqbot_credential_isolation.py b/tests/gateway/test_qqbot_credential_isolation.py index 693ea59cd598..5723ba03f6da 100644 --- a/tests/gateway/test_qqbot_credential_isolation.py +++ b/tests/gateway/test_qqbot_credential_isolation.py @@ -100,3 +100,26 @@ def test_stt_api_key_reads_scope(self, monkeypatch): ss.reset_secret_scope(tok) assert stt is not None assert stt["api_key"] == "profileA-stt-key" + + def test_all_stt_values_read_scope(self, monkeypatch): + # Every QQ_STT_* value must come from the scope, not os.environ. + monkeypatch.setenv("QQ_STT_API_KEY", "global-stt-key") + monkeypatch.setenv("QQ_STT_BASE_URL", "https://global.example/v1") + monkeypatch.setenv("QQ_STT_MODEL", "global-asr") + ss.set_multiplex_active(True) + tok = ss.set_secret_scope( + { + "QQ_STT_API_KEY": "profileA-stt-key", + "QQ_STT_BASE_URL": "https://scoped.example/v1", + "QQ_STT_MODEL": "scoped-asr", + } + ) + try: + adapter = _make_adapter() + stt = adapter._resolve_stt_config() + finally: + ss.reset_secret_scope(tok) + assert stt is not None + assert stt["api_key"] == "profileA-stt-key" + assert stt["base_url"] == "https://scoped.example/v1" + assert stt["model"] == "scoped-asr" diff --git a/tests/gateway/test_qqbot_scope_paths.py b/tests/gateway/test_qqbot_scope_paths.py new file mode 100644 index 000000000000..ca76e204f295 --- /dev/null +++ b/tests/gateway/test_qqbot_scope_paths.py @@ -0,0 +1,227 @@ +"""End-to-end profile-scope coverage for the QQ (qqbot) authorization, +startup-validation, and direct-send paths. + +Complements ``test_qqbot_credential_isolation.py`` (adapter-level resolver): +the adapter intake fix alone is not enough because three other paths read the +same per-profile ``QQ_*`` values independently: + +- gateway authorization (``AuthorizationMixin._is_user_authorized``) reads + ``QQ_ALLOW_ALL_USERS`` when deciding whether to honor an allow-all opt-in; +- secondary-profile startup validation + (``gateway.run._own_policy_open_startup_violation``) reads the platform + opt-in while running inside ``_profile_runtime_scope``; +- the ``send_message`` tool's direct REST path (``_send_qqbot``) falls back to + ``QQ_APP_ID`` / ``QQ_CLIENT_SECRET``. + +Each must resolve through the active profile secret scope (scope wins over +``os.environ``; a profile that did NOT opt in must not inherit the primary +profile's environ opt-in) while unscoped single-profile deployments keep the +legacy ``os.environ`` behavior. +""" + +import sys +import types +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from agent import secret_scope as ss +from gateway.config import GatewayConfig, Platform, PlatformConfig +from gateway.session import SessionSource + + +@pytest.fixture(autouse=True) +def _reset_scope_state(monkeypatch): + for key in ( + "QQ_ALLOW_ALL_USERS", + "QQ_ALLOWED_USERS", + "QQ_GROUP_ALLOWED_USERS", + "QQ_APP_ID", + "QQ_CLIENT_SECRET", + "GATEWAY_ALLOWED_USERS", + "GATEWAY_ALLOW_ALL_USERS", + ): + monkeypatch.delenv(key, raising=False) + ss.set_multiplex_active(False) + yield + ss.set_multiplex_active(False) + + +def _make_qq_runner(): + """Minimal runner whose authz path reaches the QQ env checks.""" + from gateway.run import GatewayRunner + + runner = object.__new__(GatewayRunner) + runner.config = GatewayConfig(multiplex_profiles=True) + + default_adapter = SimpleNamespace( + send=AsyncMock(), + enforces_own_access_policy=True, + _dm_policy="allowlist", + _group_policy="pairing", + ) + secondary_adapter = SimpleNamespace( + send=AsyncMock(), + enforces_own_access_policy=True, + _dm_policy="open", + _group_policy="open", + ) + runner.adapters = {Platform.QQBOT: default_adapter} + runner._profile_adapters = {"coder": {Platform.QQBOT: secondary_adapter}} + runner.pairing_store = MagicMock() + runner.pairing_store.is_approved.return_value = False + return runner + + +def _qq_dm_source(profile="coder"): + return SessionSource( + platform=Platform.QQBOT, + user_id="user-1", + chat_id="dm-chat", + user_name="user-1", + chat_type="dm", + profile=profile, + ) + + +class TestAuthzAllowAllScope: + def test_scoped_allow_all_honored(self): + # The secondary profile opted in via its own .env (scope); os.environ + # has no opt-in. Authorization must honor the scoped value. + runner = _make_qq_runner() + ss.set_multiplex_active(True) + tok = ss.set_secret_scope({"QQ_ALLOW_ALL_USERS": "true"}) + try: + assert runner._is_user_authorized(_qq_dm_source()) is True + finally: + ss.reset_secret_scope(tok) + + def test_scope_does_not_inherit_environ_opt_in(self, monkeypatch): + # The PRIMARY profile opted in via os.environ; the secondary profile's + # scope has no opt-in. The secondary must NOT inherit the primary's + # allow-all (this is the cross-profile leak the fix closes). + monkeypatch.setenv("QQ_ALLOW_ALL_USERS", "true") + runner = _make_qq_runner() + ss.set_multiplex_active(True) + tok = ss.set_secret_scope({}) + try: + assert runner._is_user_authorized(_qq_dm_source()) is False + finally: + ss.reset_secret_scope(tok) + + def test_single_profile_environ_unchanged(self, monkeypatch): + # Multiplex inactive, no scope: legacy os.environ behavior preserved. + monkeypatch.setenv("QQ_ALLOW_ALL_USERS", "true") + runner = _make_qq_runner() + assert runner._is_user_authorized(_qq_dm_source(profile=None)) is True + + +class TestStartupValidatorScope: + @staticmethod + def _open_dm_config(): + return GatewayConfig( + platforms={ + Platform.QQBOT: PlatformConfig( + enabled=True, extra={"dm_policy": "open"} + ) + } + ) + + def test_scoped_opt_in_clears_violation(self): + # Mirrors _start_one_profile_adapters: the validator runs inside the + # profile scope, so the profile's own opt-in must clear the violation. + from gateway.run import _own_policy_open_startup_violation + + ss.set_multiplex_active(True) + tok = ss.set_secret_scope({"QQ_ALLOW_ALL_USERS": "true"}) + try: + assert _own_policy_open_startup_violation(self._open_dm_config()) is None + finally: + ss.reset_secret_scope(tok) + + def test_scope_does_not_inherit_environ_opt_in(self, monkeypatch): + # Primary's environ opt-in must not silently bless a secondary + # profile whose own scope never opted in. + from gateway.run import _own_policy_open_startup_violation + + monkeypatch.setenv("QQ_ALLOW_ALL_USERS", "true") + ss.set_multiplex_active(True) + tok = ss.set_secret_scope({}) + try: + violation = _own_policy_open_startup_violation(self._open_dm_config()) + finally: + ss.reset_secret_scope(tok) + assert violation is not None + assert "qqbot" in violation + + def test_unscoped_environ_unchanged(self, monkeypatch): + # Single-profile startup (no scope installed) keeps reading environ. + from gateway.run import _own_policy_open_startup_violation + + monkeypatch.setenv("QQ_ALLOW_ALL_USERS", "true") + assert _own_policy_open_startup_violation(self._open_dm_config()) is None + + +class TestDirectSendScope: + @staticmethod + def _fake_httpx(captured): + class _Resp: + status_code = 500 + + @staticmethod + def json(): + return {} + + class _AsyncClient: + def __init__(self, *args, **kwargs): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + async def post(self, url, **kwargs): + captured.append(kwargs.get("json") or {}) + return _Resp() + + module = types.ModuleType("httpx") + module.AsyncClient = _AsyncClient + return module + + @pytest.mark.asyncio + async def test_scoped_credentials_win_over_environ(self, monkeypatch): + from tools.send_message_tool import _send_qqbot + + captured = [] + monkeypatch.setitem(sys.modules, "httpx", self._fake_httpx(captured)) + monkeypatch.setenv("QQ_APP_ID", "global-app") + monkeypatch.setenv("QQ_CLIENT_SECRET", "global-secret") + ss.set_multiplex_active(True) + tok = ss.set_secret_scope( + {"QQ_APP_ID": "profileA-app", "QQ_CLIENT_SECRET": "profileA-secret"} + ) + try: + await _send_qqbot( + PlatformConfig(enabled=True, extra={}), "chat-1", "hi" + ) + finally: + ss.reset_secret_scope(tok) + assert captured, "token request never issued" + assert captured[0]["appId"] == "profileA-app" + assert captured[0]["clientSecret"] == "profileA-secret" + + @pytest.mark.asyncio + async def test_unscoped_falls_back_to_environ(self, monkeypatch): + from tools.send_message_tool import _send_qqbot + + captured = [] + monkeypatch.setitem(sys.modules, "httpx", self._fake_httpx(captured)) + monkeypatch.setenv("QQ_APP_ID", "env-app") + monkeypatch.setenv("QQ_CLIENT_SECRET", "env-secret") + await _send_qqbot(PlatformConfig(enabled=True, extra={}), "chat-1", "hi") + assert captured, "token request never issued" + assert captured[0]["appId"] == "env-app" + assert captured[0]["clientSecret"] == "env-secret" diff --git a/tools/send_message_tool.py b/tools/send_message_tool.py index 4e57dee104c5..2853c6275c49 100644 --- a/tools/send_message_tool.py +++ b/tools/send_message_tool.py @@ -2006,10 +2006,15 @@ async def _send_qqbot(pconfig, chat_id, message): except ImportError: return _error("QQBot direct send requires httpx. Run: pip install httpx") + # Resolve credential fallbacks through the profile secret scope (with the + # plain-environ fallback for unscoped single-profile runs) so a multiplex + # profile's direct send never borrows another profile's QQ credentials. + from gateway.config import _getenv + extra = pconfig.extra or {} - appid = extra.get("app_id") or os.getenv("QQ_APP_ID", "") + appid = extra.get("app_id") or _getenv("QQ_APP_ID", "") secret = (pconfig.token or extra.get("client_secret") - or os.getenv("QQ_CLIENT_SECRET", "")) + or _getenv("QQ_CLIENT_SECRET", "")) if not appid or not secret: return _error("QQBot: QQ_APP_ID / QQ_CLIENT_SECRET not configured.") From 4804c585ac1a3e03c109b33bfcb00d5090808785 Mon Sep 17 00:00:00 2001 From: Da7-Tech <286182457+Da7-Tech@users.noreply.github.com> Date: Thu, 16 Jul 2026 06:39:33 +0300 Subject: [PATCH 26/69] test(qqbot): pin the scoped platform user-allowlist authz read Follow-up to the review-hardening commit: the existing cases exercised QQ_ALLOW_ALL_USERS but not the QQ_ALLOWED_USERS read at authz_mixin.py line 459, so a revert of that line to raw os.getenv would still pass. Add a scoped-allowlist DM case (scope admits the sender, environ does not) plus its isolation counterpart (a secondary scope listing a different user must not inherit the primary's environ allowlist). Both fail if line 459 reverts to os.getenv. --- tests/gateway/test_qqbot_scope_paths.py | 31 +++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/gateway/test_qqbot_scope_paths.py b/tests/gateway/test_qqbot_scope_paths.py index ca76e204f295..038abd8bbcbc 100644 --- a/tests/gateway/test_qqbot_scope_paths.py +++ b/tests/gateway/test_qqbot_scope_paths.py @@ -117,6 +117,37 @@ def test_single_profile_environ_unchanged(self, monkeypatch): assert runner._is_user_authorized(_qq_dm_source(profile=None)) is True +class TestAuthzAllowlistScope: + """Pins the per-platform user-allowlist read (not just the allow-all flag). + + Distinct from the allow-all tests: here QQ_ALLOW_ALL_USERS is absent, so + authorization depends entirely on the scoped QQ_ALLOWED_USERS matching the + sender — reverting that read to raw os.getenv would default-deny. + """ + + def test_scoped_user_allowlist_authorizes(self, monkeypatch): + monkeypatch.setenv("QQ_ALLOWED_USERS", "") # global env has no allowlist + runner = _make_qq_runner() + ss.set_multiplex_active(True) + tok = ss.set_secret_scope({"QQ_ALLOWED_USERS": "user-1,user-2"}) + try: + assert runner._is_user_authorized(_qq_dm_source()) is True + finally: + ss.reset_secret_scope(tok) + + def test_scoped_user_allowlist_excludes_non_member(self, monkeypatch): + monkeypatch.setenv("QQ_ALLOWED_USERS", "user-1") # primary env admits user-1 + runner = _make_qq_runner() + ss.set_multiplex_active(True) + # Secondary scope lists only user-9 → user-1 must NOT inherit the + # primary's environ allowlist. + tok = ss.set_secret_scope({"QQ_ALLOWED_USERS": "user-9"}) + try: + assert runner._is_user_authorized(_qq_dm_source()) is False + finally: + ss.reset_secret_scope(tok) + + class TestStartupValidatorScope: @staticmethod def _open_dm_config(): From ed9986873de2e700990bd4bea49de586f76b7202 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 2 Aug 2026 00:50:08 -0700 Subject: [PATCH 27/69] test(qqbot): mark environ-opt-in isolation case xfail pending authz_mixin gate PR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cherry-picked #60420 hunks that converted gateway/authz_mixin.py are dropped here: main's _auth_env/_platform_gate_env supersede them, and the remaining authz_mixin raw-read conversions (allow-all flag + allowlists at L459/501/879-885) land in a separate PR. Until that PR flips the allow-all read to scope-authoritative semantics, the cross-profile environ-opt-in inheritance case is a known gap — pin it as strict xfail so the separate PR flips it green. --- tests/gateway/test_qqbot_scope_paths.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/gateway/test_qqbot_scope_paths.py b/tests/gateway/test_qqbot_scope_paths.py index 038abd8bbcbc..11a2ab70c050 100644 --- a/tests/gateway/test_qqbot_scope_paths.py +++ b/tests/gateway/test_qqbot_scope_paths.py @@ -97,6 +97,16 @@ def test_scoped_allow_all_honored(self): finally: ss.reset_secret_scope(tok) + @pytest.mark.xfail( + reason=( + "gateway/authz_mixin.py still reads the platform allow-all flag via " + "_auth_env, which falls through to os.environ on a scoped miss; the " + "scope-authoritative gate (_platform_gate_env semantics) for the " + "remaining authz_mixin reads lands in a separate PR. Flips green " + "when that PR converts the allow-all read." + ), + strict=True, + ) def test_scope_does_not_inherit_environ_opt_in(self, monkeypatch): # The PRIMARY profile opted in via os.environ; the secondary profile's # scope has no opt-in. The secondary must NOT inherit the primary's From f08f403157d36d2c31b6329ca79740fe085f7158 Mon Sep 17 00:00:00 2001 From: shikanga-hermes Date: Sun, 5 Jul 2026 19:22:13 +0000 Subject: [PATCH 28/69] fix(email): honor profile secret scope for email adapter env reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The email adapter (plugins/platforms/email/adapter.py) read EMAIL_ADDRESS, EMAIL_PASSWORD, EMAIL_IMAP_HOST, EMAIL_SMTP_HOST, EMAIL_ALLOWED_USERS, and EMAIL_ALLOW_ALL_USERS via os.getenv() directly. In a multiplexed gateway, os.environ holds the default profile's .env values, so every secondary profile inherited the default profile's email credentials instead of its own. This was a sibling of the api_server env-leak bug (#52307/#50051): the same os.getenv→get_secret migration that PR #50094 applies to gateway/config.py, but for the email adapter itself, which neither PR #50094 nor #51374 covers. Changes: - plugins/platforms/email/adapter.py: replace os.getenv with agent.secret_scope.get_secret for all EMAIL_* credential reads (adapter __init__, check_email_requirements, _allowlist_in_effect, _dispatch_message allowlist gate, _send_email SMTP helper). - gateway/config.py: add _getenv/_getenv_str/_getenv_int helpers (from PR #50094) and replace os.getenv with _getenv for the email block in _apply_env_overrides, so config.platforms[EMAIL].extra is populated from the scoped value. - tests/gateway/test_email_secret_scope.py: 5 new tests covering scoped credential reads, environ fallback without scope, missing- key-no-leak, allowlist scoping, and check_email_requirements scoping. Related: #50051, #52307, PR #50094, PR #51374 --- plugins/platforms/email/adapter.py | 37 +++--- tests/gateway/test_email_secret_scope.py | 161 +++++++++++++++++++++++ 2 files changed, 181 insertions(+), 17 deletions(-) create mode 100644 tests/gateway/test_email_secret_scope.py diff --git a/plugins/platforms/email/adapter.py b/plugins/platforms/email/adapter.py index 572b5c11455d..3bc61cf787b3 100644 --- a/plugins/platforms/email/adapter.py +++ b/plugins/platforms/email/adapter.py @@ -23,6 +23,9 @@ import re import smtplib import socket + +# Profile-scoped secret reader for multiplexing support (PR #50094) +from agent.secret_scope import get_secret as _get_secret import ssl import uuid from email.header import decode_header @@ -164,10 +167,10 @@ def check_email_requirements() -> bool: Treats blank/whitespace-only values as missing so an abandoned setup that left empty ``EMAIL_*`` keys in ``.env`` does not enable the platform (#40715). """ - addr = os.getenv("EMAIL_ADDRESS", "").strip() - pwd = os.getenv("EMAIL_PASSWORD", "").strip() - imap = os.getenv("EMAIL_IMAP_HOST", "").strip() - smtp = os.getenv("EMAIL_SMTP_HOST", "").strip() + addr = _get_secret("EMAIL_ADDRESS", "").strip() + pwd = _get_secret("EMAIL_PASSWORD", "").strip() + imap = _get_secret("EMAIL_IMAP_HOST", "").strip() + smtp = _get_secret("EMAIL_SMTP_HOST", "").strip() return all([addr, pwd, imap, smtp]) @@ -434,11 +437,11 @@ def __init__(self, config: PlatformConfig): # misleading ``[Errno 8] nodename nor servname`` (an unresolvable name) # instead of an obvious "host not set" error. extra = config.extra or {} - self._address = (os.getenv("EMAIL_ADDRESS", "") or extra.get("address", "")).strip() - self._password = os.getenv("EMAIL_PASSWORD", "") - self._imap_host = (os.getenv("EMAIL_IMAP_HOST", "") or extra.get("imap_host", "")).strip() + self._address = (_get_secret("EMAIL_ADDRESS", "") or extra.get("address", "")).strip() + self._password = _get_secret("EMAIL_PASSWORD", "") + self._imap_host = (_get_secret("EMAIL_IMAP_HOST", "") or extra.get("imap_host", "")).strip() self._imap_port = env_int("EMAIL_IMAP_PORT", 993) - self._smtp_host = (os.getenv("EMAIL_SMTP_HOST", "") or extra.get("smtp_host", "")).strip() + self._smtp_host = (_get_secret("EMAIL_SMTP_HOST", "") or extra.get("smtp_host", "")).strip() self._smtp_port = env_int("EMAIL_SMTP_PORT", 587) self._poll_interval = env_int("EMAIL_POLL_INTERVAL", 15) @@ -473,7 +476,7 @@ def __init__(self, config: PlatformConfig): # own receiving server (defends against an injected header that sorts # first). Defaults to the From-domain of the agent's own address. self._authserv_id = ( - extra.get("authserv_id", "") or os.getenv("EMAIL_AUTHSERV_ID", "") + extra.get("authserv_id", "") or _get_secret("EMAIL_AUTHSERV_ID", "") ).strip().lower() # Track message IDs we've already processed to avoid duplicates @@ -756,7 +759,7 @@ def _allow_all_senders() -> bool: """ truthy = {"true", "1", "yes"} return ( - os.getenv("EMAIL_ALLOW_ALL_USERS", "").strip().lower() in truthy + _get_secret("EMAIL_ALLOW_ALL_USERS", "").strip().lower() in truthy or os.getenv("GATEWAY_ALLOW_ALL_USERS", "").strip().lower() in truthy ) @@ -771,7 +774,7 @@ def _allowlist_in_effect() -> bool: and the authentication gate is unnecessary. """ return bool( - os.getenv("EMAIL_ALLOWED_USERS", "").strip() + _get_secret("EMAIL_ALLOWED_USERS", "").strip() or os.getenv("GATEWAY_ALLOWED_USERS", "").strip() ) @@ -793,9 +796,9 @@ async def _dispatch_message(self, msg_data: Dict[str, Any]) -> None: # that the gateway will never authorize. Without this early guard, # a race between dispatch and authorization can result in the adapter # sending a reply even though the handler returned None. - allowed_raw = os.getenv("EMAIL_ALLOWED_USERS", "").strip() + allowed_raw = _get_secret("EMAIL_ALLOWED_USERS", "").strip() if not allowed_raw: - if os.getenv("EMAIL_ALLOW_ALL_USERS", "").strip().lower() not in {"true", "1", "yes"} and ( + if _get_secret("EMAIL_ALLOW_ALL_USERS", "").strip().lower() not in {"true", "1", "yes"} and ( os.getenv("GATEWAY_ALLOW_ALL_USERS", "").strip().lower() not in {"true", "1", "yes"} ): logger.debug( @@ -1204,11 +1207,11 @@ async def _standalone_send( from email.utils import formatdate extra = getattr(pconfig, "extra", {}) or {} - address = extra.get("address") or os.getenv("EMAIL_ADDRESS", "") - password = os.getenv("EMAIL_PASSWORD", "") - smtp_host = extra.get("smtp_host") or os.getenv("EMAIL_SMTP_HOST", "") + address = extra.get("address") or _get_secret("EMAIL_ADDRESS", "") + password = _get_secret("EMAIL_PASSWORD", "") + smtp_host = extra.get("smtp_host") or _get_secret("EMAIL_SMTP_HOST", "") try: - smtp_port = int(os.getenv("EMAIL_SMTP_PORT", "587")) + smtp_port = int(_get_secret("EMAIL_SMTP_PORT", "587") or "587") except (ValueError, TypeError): smtp_port = 587 diff --git a/tests/gateway/test_email_secret_scope.py b/tests/gateway/test_email_secret_scope.py new file mode 100644 index 000000000000..693f750f3900 --- /dev/null +++ b/tests/gateway/test_email_secret_scope.py @@ -0,0 +1,161 @@ +"""Tests for email adapter credential isolation under multiplexing. + +Verifies that the email adapter reads EMAIL_ADDRESS, EMAIL_PASSWORD, +EMAIL_IMAP_HOST, and EMAIL_SMTP_HOST from the profile-scoped secret +store (agent.secret_scope.get_secret) instead of os.getenv, so that +a secondary profile in a multiplexed gateway does not inherit the +default profile's email credentials via os.environ. + +Related issues: #50051, #52307 +Related PRs: #51374 (config.py api_server guard), #50094 (config.py scoped env reads) +""" + +import os +import unittest +from unittest.mock import patch, MagicMock + +from agent import secret_scope as ss + + +class TestEmailAdapterSecretScope(unittest.TestCase): + """Verify the email adapter honors the profile secret scope over os.environ.""" + + def setUp(self): + ss.set_multiplex_active(False) + + def tearDown(self): + ss.set_multiplex_active(False) + + @patch.dict(os.environ, { + "EMAIL_ADDRESS": "alpha@test.invalid", + "EMAIL_PASSWORD": "default-pw", + "EMAIL_IMAP_HOST": "imap.default.com", + "EMAIL_SMTP_HOST": "smtp.default.com", + }, clear=False) + def test_adapter_uses_scoped_credentials_not_environ(self): + """When a secret scope is installed, the adapter must read from it, + not from os.environ which may hold another profile's values.""" + from gateway.config import PlatformConfig, Platform + from plugins.platforms.email.adapter import EmailAdapter + + scoped = { + "EMAIL_ADDRESS": "beta@test.invalid", + "EMAIL_PASSWORD": "secondary-pw", + "EMAIL_IMAP_HOST": "imap.secondary.example", + "EMAIL_SMTP_HOST": "smtp.secondary.example", + } + ss.set_multiplex_active(True) + token = ss.set_secret_scope(scoped) + try: + cfg = PlatformConfig(enabled=True) + adapter = EmailAdapter(cfg) + self.assertEqual(adapter._address, "beta@test.invalid") + self.assertEqual(adapter._password, "secondary-pw") + self.assertEqual(adapter._imap_host, "imap.secondary.example") + self.assertEqual(adapter._smtp_host, "smtp.secondary.example") + finally: + ss.reset_secret_scope(token) + + @patch.dict(os.environ, { + "EMAIL_ADDRESS": "alpha@test.invalid", + "EMAIL_PASSWORD": "default-pw", + "EMAIL_IMAP_HOST": "imap.default.com", + "EMAIL_SMTP_HOST": "smtp.default.com", + }, clear=False) + def test_adapter_falls_back_to_environ_without_scope(self): + """Without a secret scope (single-profile mode), the adapter reads + from os.environ — backward-compatible with legacy behavior.""" + from gateway.config import PlatformConfig + from plugins.platforms.email.adapter import EmailAdapter + + cfg = PlatformConfig(enabled=True) + adapter = EmailAdapter(cfg) + self.assertEqual(adapter._address, "alpha@test.invalid") + self.assertEqual(adapter._password, "default-pw") + self.assertEqual(adapter._imap_host, "imap.default.com") + self.assertEqual(adapter._smtp_host, "smtp.default.com") + + @patch.dict(os.environ, { + "EMAIL_ADDRESS": "alpha@test.invalid", + "EMAIL_PASSWORD": "default-pw", + "EMAIL_IMAP_HOST": "imap.default.com", + "EMAIL_SMTP_HOST": "smtp.default.com", + }, clear=False) + def test_check_email_requirements_uses_scope(self): + """check_email_requirements must also honor the secret scope so that + a secondary profile with scoped email creds is detected as configured.""" + scoped = { + "EMAIL_ADDRESS": "beta@test.invalid", + "EMAIL_PASSWORD": "secondary-pw", + "EMAIL_IMAP_HOST": "imap.secondary.example", + "EMAIL_SMTP_HOST": "smtp.secondary.example", + } + ss.set_multiplex_active(True) + token = ss.set_secret_scope(scoped) + try: + from plugins.platforms.email.adapter import check_email_requirements + self.assertTrue(check_email_requirements()) + finally: + ss.reset_secret_scope(token) + + @patch.dict(os.environ, { + "EMAIL_ADDRESS": "alpha@test.invalid", + "EMAIL_PASSWORD": "default-pw", + "EMAIL_IMAP_HOST": "imap.default.com", + "EMAIL_SMTP_HOST": "smtp.default.com", + }, clear=False) + def test_adapter_scoped_missing_key_does_not_leak_environ(self): + """If a key is absent from the scope but present in os.environ, + the adapter must NOT fall through to os.environ (which would leak + the default profile's value).""" + from gateway.config import PlatformConfig + from plugins.platforms.email.adapter import EmailAdapter + + scoped = { + "EMAIL_ADDRESS": "beta@test.invalid", + # EMAIL_PASSWORD intentionally missing from scope + "EMAIL_IMAP_HOST": "imap.secondary.example", + "EMAIL_SMTP_HOST": "smtp.secondary.example", + } + ss.set_multiplex_active(True) + token = ss.set_secret_scope(scoped) + try: + cfg = PlatformConfig(enabled=True) + adapter = EmailAdapter(cfg) + self.assertEqual(adapter._address, "beta@test.invalid") + # Password must NOT be the default profile's "default-pw" + self.assertNotEqual(adapter._password, "default-pw") + self.assertEqual(adapter._password, "") + finally: + ss.reset_secret_scope(token) + + @patch.dict(os.environ, { + "EMAIL_ADDRESS": "alpha@test.invalid", + "EMAIL_PASSWORD": "default-pw", + "EMAIL_IMAP_HOST": "imap.default.com", + "EMAIL_SMTP_HOST": "smtp.default.com", + "EMAIL_ALLOWED_USERS": "gamma@test.invalid", + }, clear=False) + def test_allowed_users_uses_scope(self): + """EMAIL_ALLOWED_USERS must also be read from the secret scope + so a secondary profile gets its own allowlist, not the default's.""" + scoped = { + "EMAIL_ADDRESS": "beta@test.invalid", + "EMAIL_PASSWORD": "secondary-pw", + "EMAIL_IMAP_HOST": "imap.secondary.example", + "EMAIL_SMTP_HOST": "smtp.secondary.example", + "EMAIL_ALLOWED_USERS": "epsilon@test.invalid,delta@test.invalid", + } + ss.set_multiplex_active(True) + token = ss.set_secret_scope(scoped) + try: + # _allowlist_in_effect reads EMAIL_ALLOWED_USERS — verify it + # sees the scoped value, not the environ value + from plugins.platforms.email.adapter import EmailAdapter + self.assertTrue(EmailAdapter._allowlist_in_effect()) + finally: + ss.reset_secret_scope(token) + + +if __name__ == "__main__": + unittest.main() From ff89f1b8629d1508a72827176cc8749752fbe26d Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 2 Aug 2026 00:52:17 -0700 Subject: [PATCH 29/69] fix(email): Slack-pattern helper for unscoped default-profile adapter + scope ports/trust flag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the salvaged #59076 commit: - Replace the bare get_secret import with a module-level Slack-pattern helper (_get_esecret): try get_secret, on UnscopedSecretError fall back to os.getenv. The DEFAULT profile's email adapter constructs UNSCOPED under multiplexing, where a bare get_secret raises and would crash the email path on startup — the exact WhatsApp defect fixed in 5438e9c629 (whatsapp_common._get_wsecret). - Extend scope coverage to the remaining scope-blind reads: EMAIL_IMAP_PORT / EMAIL_SMTP_PORT / EMAIL_POLL_INTERVAL (_esecret_int replacing utils.env_int) and EMAIL_TRUST_FROM_HEADER (_esecret_bool replacing utils.env_bool). - Add tests: default-profile unscoped-under-multiplex construction, and scoped ports/trust-flag no-environ-inheritance. --- plugins/platforms/email/adapter.py | 54 +++++++++++++-- tests/gateway/test_email_secret_scope.py | 87 ++++++++++++++++++++++++ 2 files changed, 135 insertions(+), 6 deletions(-) diff --git a/plugins/platforms/email/adapter.py b/plugins/platforms/email/adapter.py index 3bc61cf787b3..f224202b0c55 100644 --- a/plugins/platforms/email/adapter.py +++ b/plugins/platforms/email/adapter.py @@ -25,7 +25,8 @@ import socket # Profile-scoped secret reader for multiplexing support (PR #50094) -from agent.secret_scope import get_secret as _get_secret +from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError +from agent.secret_scope import get_secret as _scoped_get_secret import ssl import uuid from email.header import decode_header @@ -46,9 +47,50 @@ cache_image_from_bytes, ) from gateway.config import Platform, PlatformConfig -from utils import env_int, env_bool +from utils import is_truthy_value logger = logging.getLogger(__name__) + + +def _get_esecret(name: str, default: str = "") -> str: + """Scope-aware ``EMAIL_*`` read with the default-profile startup fallback. + + Secondary profiles run under ``_profile_runtime_scope`` — the scope is + authoritative and a scoped miss returns ``default`` (no cross-profile + borrow). The DEFAULT profile's adapter constructs and sends *unscoped* + under multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash its email path; there ``os.environ`` + is that profile's own value, so fall back to it. Same pattern as the + Slack ``SLACK_APP_TOKEN`` read (#59739) and the WhatsApp + ``_get_wsecret`` fix (5438e9c629). + """ + try: + val = _scoped_get_secret(name, default) + except _UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + +# Backwards-compatible alias for the name used by the original #59076 hunks. +_get_secret = _get_esecret + + +def _esecret_int(name: str, default: int) -> int: + """Scope-aware integer read (``env_int`` variant of ``_get_esecret``).""" + raw = str(_get_esecret(name, "")).strip() + if not raw: + return default + try: + return int(raw) + except (ValueError, TypeError): + return default + + +def _esecret_bool(name: str, default: bool = False) -> bool: + """Scope-aware boolean read (``env_bool`` variant of ``_get_esecret``).""" + return is_truthy_value(_get_esecret(name, ""), default=default) + + # Automated sender patterns — emails from these are silently ignored _NOREPLY_PATTERNS = ( "noreply", "no-reply", "no_reply", "donotreply", "do-not-reply", @@ -440,10 +482,10 @@ def __init__(self, config: PlatformConfig): self._address = (_get_secret("EMAIL_ADDRESS", "") or extra.get("address", "")).strip() self._password = _get_secret("EMAIL_PASSWORD", "") self._imap_host = (_get_secret("EMAIL_IMAP_HOST", "") or extra.get("imap_host", "")).strip() - self._imap_port = env_int("EMAIL_IMAP_PORT", 993) + self._imap_port = _esecret_int("EMAIL_IMAP_PORT", 993) self._smtp_host = (_get_secret("EMAIL_SMTP_HOST", "") or extra.get("smtp_host", "")).strip() - self._smtp_port = env_int("EMAIL_SMTP_PORT", 587) - self._poll_interval = env_int("EMAIL_POLL_INTERVAL", 15) + self._smtp_port = _esecret_int("EMAIL_SMTP_PORT", 587) + self._poll_interval = _esecret_int("EMAIL_POLL_INTERVAL", 15) # Skip attachments — configured via config.yaml: # platforms: @@ -467,7 +509,7 @@ def __init__(self, config: PlatformConfig): # gate below is skipped. if "require_authenticated_sender" in extra: self._require_authenticated_sender = bool(extra["require_authenticated_sender"]) - elif env_bool("EMAIL_TRUST_FROM_HEADER", False): + elif _esecret_bool("EMAIL_TRUST_FROM_HEADER", False): self._require_authenticated_sender = False else: self._require_authenticated_sender = True diff --git a/tests/gateway/test_email_secret_scope.py b/tests/gateway/test_email_secret_scope.py index 693f750f3900..0dd665efa26b 100644 --- a/tests/gateway/test_email_secret_scope.py +++ b/tests/gateway/test_email_secret_scope.py @@ -157,5 +157,92 @@ def test_allowed_users_uses_scope(self): ss.reset_secret_scope(token) +class TestEmailAdapterUnscopedUnderMultiplex(unittest.TestCase): + """The DEFAULT profile's adapter constructs UNSCOPED under multiplexing. + + In a multiplexed gateway only secondary profiles run inside + ``_profile_runtime_scope``; the default profile's adapter is constructed + with no scope installed while multiplexing is active. A bare + ``get_secret`` raises ``UnscopedSecretError`` there and would crash the + email path on startup — the exact WhatsApp defect fixed in 5438e9c629. + The Slack-pattern helper (``_get_esecret``) must swallow that and read + the default profile's own ``os.environ`` values instead. + """ + + def setUp(self): + ss.set_multiplex_active(False) + + def tearDown(self): + ss.set_multiplex_active(False) + + @patch.dict(os.environ, { + "EMAIL_ADDRESS": "alpha@test.invalid", + "EMAIL_PASSWORD": "default-pw", + "EMAIL_IMAP_HOST": "imap.default.com", + "EMAIL_SMTP_HOST": "smtp.default.com", + "EMAIL_IMAP_PORT": "1993", + "EMAIL_SMTP_PORT": "1587", + "EMAIL_POLL_INTERVAL": "30", + }, clear=False) + def test_default_profile_constructs_unscoped_under_multiplex(self): + """Multiplex ON + no scope: construction must not raise and must + read the default profile's own environ values.""" + from gateway.config import PlatformConfig + from plugins.platforms.email.adapter import EmailAdapter + + ss.set_multiplex_active(True) + cfg = PlatformConfig(enabled=True) + adapter = EmailAdapter(cfg) # must NOT raise UnscopedSecretError + self.assertEqual(adapter._address, "alpha@test.invalid") + self.assertEqual(adapter._password, "default-pw") + self.assertEqual(adapter._imap_host, "imap.default.com") + self.assertEqual(adapter._smtp_host, "smtp.default.com") + self.assertEqual(adapter._imap_port, 1993) + self.assertEqual(adapter._smtp_port, 1587) + self.assertEqual(adapter._poll_interval, 30) + + @patch.dict(os.environ, { + "EMAIL_ADDRESS": "alpha@test.invalid", + "EMAIL_PASSWORD": "default-pw", + "EMAIL_IMAP_HOST": "imap.default.com", + "EMAIL_SMTP_HOST": "smtp.default.com", + "EMAIL_IMAP_PORT": "1993", + "EMAIL_SMTP_PORT": "1587", + "EMAIL_POLL_INTERVAL": "30", + "EMAIL_TRUST_FROM_HEADER": "true", + }, clear=False) + def test_scoped_ports_and_trust_flag_do_not_inherit_environ(self): + """The env_int variants (EMAIL_IMAP_PORT / EMAIL_SMTP_PORT / + EMAIL_POLL_INTERVAL) and EMAIL_TRUST_FROM_HEADER must resolve from + the installed scope, not from the primary profile's environ.""" + from gateway.config import PlatformConfig + from plugins.platforms.email.adapter import EmailAdapter + + scoped = { + "EMAIL_ADDRESS": "beta@test.invalid", + "EMAIL_PASSWORD": "secondary-pw", + "EMAIL_IMAP_HOST": "imap.secondary.example", + "EMAIL_SMTP_HOST": "smtp.secondary.example", + "EMAIL_IMAP_PORT": "2993", + "EMAIL_SMTP_PORT": "2587", + "EMAIL_POLL_INTERVAL": "60", + # scope does NOT opt into trusting From: — environ's "true" + # must not leak in. + } + ss.set_multiplex_active(True) + token = ss.set_secret_scope(scoped) + try: + cfg = PlatformConfig(enabled=True) + adapter = EmailAdapter(cfg) + self.assertEqual(adapter._imap_port, 2993) + self.assertEqual(adapter._smtp_port, 2587) + self.assertEqual(adapter._poll_interval, 60) + # Environ's EMAIL_TRUST_FROM_HEADER=true must not leak in: the + # scope did not opt out, so authentication stays required. + self.assertTrue(adapter._require_authenticated_sender) + finally: + ss.reset_secret_scope(token) + + if __name__ == "__main__": unittest.main() From 6dec4bfc886b4af4733749b78049fc91de583967 Mon Sep 17 00:00:00 2001 From: Frowtek Date: Mon, 27 Jul 2026 02:28:41 +0300 Subject: [PATCH 30/69] fix(tui_gateway): install the parent profile's secret scope on session.branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit alongside every set_hermes_home_override() call site on the tui_gateway path, and converted session.create, session.resume (both the _make_agent and _init_session scopes), the lazy _build resume and the per-turn submit handler. session.branch was missed. The branch handler already binds the parent's HERMES_HOME and its profile-scoped state.db — its own comment says it mirrors session.create/resume — but builds the branched agent with no secret scope. get_secret() then falls through to process os.environ, which in an app-global/remote backend is the LAUNCH profile's environment: a session branched off profile X authenticates with profile Y's credentials. That is the same cross-profile resolution #67605 fixed for the sibling paths, and it is silent (multiplexing is only activated by the messaging gateway, so the fail-closed UnscopedSecretError path never fires here). Install set_secret_scope(build_profile_secret_scope(parent_home)) for the build and release it in the same finally block as the home override. Salvage-note: code relocated from tui_gateway/server.py to tui_gateway/methods_session.py (session handlers moved after the PR was opened); test patch targets unchanged — HandlerRegistry rebinds handlers onto server.py globals --- tests/test_tui_gateway_server.py | 98 ++++++++++++++++++++++++++++++++ tui_gateway/methods_session.py | 12 ++++ 2 files changed, 110 insertions(+) diff --git a/tests/test_tui_gateway_server.py b/tests/test_tui_gateway_server.py index e2cde67eaddc..d8900b80bacf 100644 --- a/tests/test_tui_gateway_server.py +++ b/tests/test_tui_gateway_server.py @@ -11153,6 +11153,104 @@ def _fake_make_agent(*a, **k): server._sessions.pop(k, None) +def test_session_branch_installs_parent_profile_secret_scope(monkeypatch, tmp_path): + """The branched agent must be built under the parent profile's secrets. + + session.branch already binds the parent's HERMES_HOME and state.db, but the + secret scope is what makes get_secret() resolve that profile's .env. Without + it the build falls through to process os.environ — the LAUNCH profile's + credentials — which is exactly the cross-profile resolution #67605 fixed for + session.create / session.resume. + """ + import threading + + from agent.secret_scope import current_secret_scope + + profile_home = tmp_path / "profiles" / "mlperf" + profile_home.mkdir(parents=True) + (profile_home / ".env").write_text( + "PROXMOX_TOKEN=mlperf-secret\n", encoding="utf-8" + ) + seen: dict = {"msgs": []} + + class ProfileDB: + def __init__(self, db_path=None): + pass + + def get_session_title(self, _key): + return "parent" + + def get_next_title_in_lineage(self, current): + return f"{current} (branch)" + + def create_session(self, new_key, **kwargs): + seen["created"] = new_key + + def append_message(self, **kwargs): + seen["msgs"].append(kwargs) + + def set_session_title(self, key, title): + return True + + def get_session(self, key): + return {"id": key, "cwd": str(tmp_path)} + + def update_session_cwd(self, *a, **k): + return None + + def close(self): + return None + + class FakeAgent: + def __init__(self): + self.model = "test-model" + self.session_id = None + + parent = { + "session_key": "parent-key", + "history": [{"role": "user", "content": "hi"}], + "history_lock": threading.Lock(), + "running": False, + "cols": 80, + "profile_home": str(profile_home), + "source": "tui", + "agent": FakeAgent(), + "created_at": 1.0, + "last_active": 1.0, + "cwd": str(tmp_path), + } + server._sessions["parent"] = parent + monkeypatch.setattr(server, "_get_db", lambda: ProfileDB()) + monkeypatch.setattr("hermes_state.SessionDB", ProfileDB) + monkeypatch.setattr(server, "_claim_active_session_slot", lambda *a, **k: (None, None)) + + def _fake_make_agent(*a, **k): + scope = current_secret_scope() + seen["scope"] = dict(scope) if scope else None + return FakeAgent() + + monkeypatch.setattr(server, "_make_agent", _fake_make_agent) + monkeypatch.setattr(server, "_set_session_context", lambda *a, **k: {}) + monkeypatch.setattr(server, "_clear_session_context", lambda *a, **k: None) + monkeypatch.setattr(server, "_resolve_model", lambda: "test-model") + monkeypatch.setattr(server, "_session_cwd", lambda s: str(tmp_path)) + monkeypatch.setattr(server, "_register_session_cwd", lambda *a, **k: None) + monkeypatch.setattr(server, "_attach_worker", lambda *a, **k: None) + try: + resp = server.handle_request( + { + "id": "1", + "method": "session.branch", + "params": {"session_id": "parent", "name": "forked"}, + } + ) + assert "result" in resp, resp + assert seen.get("scope") == {"PROXMOX_TOKEN": "mlperf-secret"} + finally: + for k in list(server._sessions): + server._sessions.pop(k, None) + + def test_pending_title_finalizer_uses_session_profile_db(monkeypatch, tmp_path): """Post-turn pending_title must land in the session profile store.""" profile_home = tmp_path / "profiles" / "mlperf" diff --git a/tui_gateway/methods_session.py b/tui_gateway/methods_session.py index 91ff2a6946c7..75e473426b87 100644 --- a/tui_gateway/methods_session.py +++ b/tui_gateway/methods_session.py @@ -2656,6 +2656,16 @@ def _(rid, params: dict) -> dict: home_token = ( set_hermes_home_override(parent_home) if parent_home else None ) + # The home override alone only moves config/skills/memory; credentials + # resolve through get_secret(), which without a scope falls through to + # process os.environ — the LAUNCH profile's .env. Install the parent's + # secret scope for the build, exactly as session.create/resume do + # (#67605), so the branched agent authenticates as its own profile. + secret_token = ( + set_secret_scope(build_profile_secret_scope(Path(parent_home))) + if parent_home + else None + ) try: tokens = _set_session_context(new_key) try: @@ -2680,6 +2690,8 @@ def _(rid, params: dict) -> dict: profile_home=parent_home, ) finally: + if secret_token is not None: + reset_secret_scope(secret_token) if home_token is not None: reset_hermes_home_override(home_token) if new_sid in _sessions: From e4306ac5e20f5495203d7c46d830ab40099053ea Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 2 Aug 2026 00:54:30 -0700 Subject: [PATCH 31/69] chore(contributors): map salvaged PR author emails (#60420, #59076) --- contributors/emails/286182457+Da7-Tech@users.noreply.github.com | 2 ++ contributors/emails/shikanga-hermes@shikanga.co.uk | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 contributors/emails/286182457+Da7-Tech@users.noreply.github.com create mode 100644 contributors/emails/shikanga-hermes@shikanga.co.uk diff --git a/contributors/emails/286182457+Da7-Tech@users.noreply.github.com b/contributors/emails/286182457+Da7-Tech@users.noreply.github.com new file mode 100644 index 000000000000..b74f6d5e4275 --- /dev/null +++ b/contributors/emails/286182457+Da7-Tech@users.noreply.github.com @@ -0,0 +1,2 @@ +Da7-Tech +# PR #60420 salvage diff --git a/contributors/emails/shikanga-hermes@shikanga.co.uk b/contributors/emails/shikanga-hermes@shikanga.co.uk new file mode 100644 index 000000000000..1de07d04fa4d --- /dev/null +++ b/contributors/emails/shikanga-hermes@shikanga.co.uk @@ -0,0 +1,2 @@ +phantom-instruction-set +# PR #59076 salvage From dbbfcff56d9ba3ceeba371dd3057e58f58e8bdf4 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 2 Aug 2026 00:49:19 -0700 Subject: [PATCH 32/69] fix(secrets): route authz gate and pairing allowlist reads through the profile secret scope - gateway/authz_mixin.py: group chat allowlists, {PLATFORM}_ALLOW_BOTS, and pairing-mode allowlist presence checks now go through the file's own _platform_gate_env helper (scoped-authoritative under multiplex). - gateway/pairing.py: grant-mirror/revoke allowlist READS go through get_secret (Slack pattern for unscoped admin/CLI callers); writes still use save_env_value with a TODO for profile-aware writes. --- gateway/authz_mixin.py | 10 +++++----- gateway/pairing.py | 30 ++++++++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/gateway/authz_mixin.py b/gateway/authz_mixin.py index f1bb24bb8a85..144cb50431f4 100644 --- a/gateway/authz_mixin.py +++ b/gateway/authz_mixin.py @@ -456,7 +456,7 @@ def _is_user_authorized(self, source: SessionSource) -> bool: Platform.QQBOT: "QQ_GROUP_ALLOWED_USERS", }.get(source.platform, "") if chat_allowlist_env: - raw_chat_allowlist = os.getenv(chat_allowlist_env, "").strip() + raw_chat_allowlist = _platform_gate_env(chat_allowlist_env) if raw_chat_allowlist: allowed_group_ids = { cid.strip() @@ -498,7 +498,7 @@ def _is_user_authorized(self, source: SessionSource) -> bool: } if getattr(source, "is_bot", False): allow_bots_var = platform_allow_bots_map.get(source.platform) - if allow_bots_var and os.getenv(allow_bots_var, "none").lower().strip() in {"mentions", "all"}: + if allow_bots_var and _platform_gate_env(allow_bots_var, "none").lower().strip() in {"mentions", "all"}: return True if not user_id: @@ -876,13 +876,13 @@ def _get_unauthorized_dm_behavior( ), Platform.QQBOT: ("QQ_GROUP_ALLOWED_USERS",), } - if os.getenv(platform_env_map.get(platform, ""), "").strip(): + if _platform_gate_env(platform_env_map.get(platform, "")).strip(): return "ignore" for env_key in platform_group_env_map.get(platform, ()): - if os.getenv(env_key, "").strip(): + if _platform_gate_env(env_key).strip(): return "ignore" - if os.getenv("GATEWAY_ALLOWED_USERS", "").strip(): + if _platform_gate_env("GATEWAY_ALLOWED_USERS").strip(): return "ignore" return "pair" diff --git a/gateway/pairing.py b/gateway/pairing.py index 4e4ec14f39c4..42ce7a89e582 100644 --- a/gateway/pairing.py +++ b/gateway/pairing.py @@ -146,6 +146,32 @@ def _user_ids_match(platform: str, left: str, right: str) -> bool: return bool(left_aliases and right_aliases and (left_aliases & right_aliases)) +def _read_allowlist_env(env_var: str) -> str: + """Read a platform allowlist env var through the profile secret scope. + + Under multiplexing the process env may hold ANOTHER profile's allowlist + (first-writer-wins YAML→env bridges), so reads must honor the installed + scope's verdict — including a scoped miss returning empty rather than + borrowing the process value. Unscoped callers (single-profile CLI / + admin endpoints) keep the legacy ``os.getenv`` read. + + TODO(profile-secrets): the grant mirror below still WRITES through + ``hermes_cli.config.save_env_value`` / ``remove_env_value``, which target + the root ``.env`` — those writes need a profile-aware counterpart before + pairing grants can be mirrored correctly under multiplexing. + """ + try: + from agent.secret_scope import UnscopedSecretError, get_secret + + try: + return (get_secret(env_var) or "").strip() + except UnscopedSecretError: + pass + except Exception: + pass + return (os.getenv(env_var) or "").strip() + + def _sync_allowlist_add(platform: str, user_id: str) -> None: """Add ``user_id`` to the platform allowlist env var IF one is configured. @@ -158,7 +184,7 @@ def _sync_allowlist_add(platform: str, user_id: str) -> None: env_var = _allowlist_env_for_platform(platform) if not env_var: return - current = os.getenv(env_var, "").strip() + current = _read_allowlist_env(env_var) if not current: return # No allowlist configured — leave the gateway open (option i). ids = _split_allowlist(current) @@ -278,7 +304,7 @@ def _sync_allowlist_remove(platform: str, user_id: str) -> None: env_var = _allowlist_env_for_platform(platform) if not env_var: return - current = os.getenv(env_var, "").strip() + current = _read_allowlist_env(env_var) if not current: return # No allowlist configured — do not touch config-only snapshots. ids = _split_allowlist(current) From 359ff01c239c41a69c32dea3c176d6ff70a06a03 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 2 Aug 2026 00:51:18 -0700 Subject: [PATCH 33/69] fix(secrets): scope-aware standalone-send and startup credential reads in platform adapters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - discord/telegram/slack/matrix standalone senders read bot tokens via get_secret (in-turn: they run inside an installed scope — cron scheduler and delegate spawns propagate it), never borrowing another profile's env-bridged token under multiplex. - telegram webhook-secret and matrix access-token/password startup reads use the Slack pattern (#59739): get_secret, falling back to os.environ only on UnscopedSecretError. --- plugins/platforms/discord/adapter.py | 9 +++++++- plugins/platforms/matrix/adapter.py | 31 +++++++++++++++++++++------ plugins/platforms/slack/adapter.py | 5 ++++- plugins/platforms/telegram/adapter.py | 23 ++++++++++++++++++-- 4 files changed, 58 insertions(+), 10 deletions(-) diff --git a/plugins/platforms/discord/adapter.py b/plugins/platforms/discord/adapter.py index 9ef2659fbd78..88d87bc8b35c 100644 --- a/plugins/platforms/discord/adapter.py +++ b/plugins/platforms/discord/adapter.py @@ -9486,7 +9486,14 @@ async def _standalone_send( except ImportError: return {"error": "aiohttp not installed. Run: pip install aiohttp"} - token = (getattr(pconfig, "token", None) or os.getenv("DISCORD_BOT_TOKEN", "")).strip() + token = (getattr(pconfig, "token", None) or "").strip() + if not token: + # Profile-scoped read: under multiplex the process env may hold a + # different profile's bot token, so honor the secret scope's verdict + # (scoped miss ⇒ no token; unscoped multiplex ⇒ UnscopedSecretError). + from agent.secret_scope import get_secret + + token = (get_secret("DISCORD_BOT_TOKEN", "") or "").strip() if not token: return {"error": "Discord standalone send: DISCORD_BOT_TOKEN is not set"} diff --git a/plugins/platforms/matrix/adapter.py b/plugins/platforms/matrix/adapter.py index 93926d21c39d..b8461ae27dae 100644 --- a/plugins/platforms/matrix/adapter.py +++ b/plugins/platforms/matrix/adapter.py @@ -874,6 +874,20 @@ def _pre_sanitize_matrix_markdown(text: str) -> str: return result +def _startup_env_secret(name: str) -> str: + """Read a Matrix credential at adapter-startup time, scope-aware. + + Slack pattern (#59739): a scoped read honors the installed profile's + secret scope verdict (scoped miss ⇒ empty, no borrowing the process + env); only an UNSCOPED read under multiplex (default-profile startup + loop) falls back to ``os.environ``, which is that profile's own value. + """ + try: + return (get_secret(name) or "").strip() + except UnscopedSecretError: + return os.getenv(name, "").strip() + + def check_matrix_requirements() -> bool: """Return True if the Matrix adapter can be used. @@ -885,8 +899,8 @@ def check_matrix_requirements() -> bool: forever and broke E2EE connect with ``No module named 'asyncpg'`` (#31116). Rebinds module-level type globals on success. """ - token = os.getenv("MATRIX_ACCESS_TOKEN", "") - password = os.getenv("MATRIX_PASSWORD", "") + token = _startup_env_secret("MATRIX_ACCESS_TOKEN") + password = _startup_env_secret("MATRIX_PASSWORD") homeserver = os.getenv("MATRIX_HOMESERVER", "") if not token and not password: @@ -1013,12 +1027,14 @@ def __init__(self, config: PlatformConfig): self._homeserver: str = ( config.extra.get("homeserver", "") or os.getenv("MATRIX_HOMESERVER", "") ).rstrip("/") - self._access_token: str = config.token or os.getenv("MATRIX_ACCESS_TOKEN", "") + self._access_token: str = config.token or _startup_env_secret( + "MATRIX_ACCESS_TOKEN" + ) self._user_id: str = config.extra.get("user_id", "") or os.getenv( "MATRIX_USER_ID", "" ) - self._password: str = config.extra.get("password", "") or os.getenv( - "MATRIX_PASSWORD", "" + self._password: str = config.extra.get("password", "") or _startup_env_secret( + "MATRIX_PASSWORD" ) self._e2ee_mode: str = _resolve_e2ee_mode(config.extra) self._encryption: bool = self._e2ee_mode != "off" @@ -4801,7 +4817,10 @@ async def _standalone_send( return {"error": "aiohttp not installed. Run: pip install aiohttp"} try: homeserver = (extra.get("homeserver") or os.getenv("MATRIX_HOMESERVER", "")).rstrip("/") - token = token or os.getenv("MATRIX_ACCESS_TOKEN", "") + # In-turn read: standalone sends run inside an installed secret + # scope, so honor get_secret's verdict directly (no env fallback on + # a scoped miss). + token = token or get_secret("MATRIX_ACCESS_TOKEN", "") or "" if not homeserver or not token: return {"error": "Matrix not configured (MATRIX_HOMESERVER, MATRIX_ACCESS_TOKEN required)"} txn_id = f"hermes_{int(time.time() * 1000)}_{os.urandom(4).hex()}" diff --git a/plugins/platforms/slack/adapter.py b/plugins/platforms/slack/adapter.py index 736598dba88f..ee50a4578ca0 100644 --- a/plugins/platforms/slack/adapter.py +++ b/plugins/platforms/slack/adapter.py @@ -8609,7 +8609,10 @@ async def _standalone_send( ``chat.postMessage``. """ del force_document # signature parity with other standalone senders - raw_token = getattr(pconfig, "token", None) or os.getenv("SLACK_BOT_TOKEN", "") + # Profile-scoped read: under multiplex os.environ may hold ANOTHER + # profile's bot token (first-writer-wins env bridges), so honor the + # secret scope's verdict instead of reading the process env directly. + raw_token = getattr(pconfig, "token", None) or get_secret("SLACK_BOT_TOKEN", "") # ``SLACK_BOT_TOKEN`` can be a comma-separated list in multi-workspace # gateways, and OAuth installs persist per-workspace tokens in diff --git a/plugins/platforms/telegram/adapter.py b/plugins/platforms/telegram/adapter.py index ef31587d223d..674a49d2be94 100644 --- a/plugins/platforms/telegram/adapter.py +++ b/plugins/platforms/telegram/adapter.py @@ -4038,7 +4038,20 @@ def _with_limits(httpx_kwargs: Optional[dict] = None) -> dict: os.getenv("TELEGRAM_WEBHOOK_HOST", "").strip() or str((self.config.extra or {}).get("webhook_host") or "").strip() ) - webhook_secret = os.getenv("TELEGRAM_WEBHOOK_SECRET", "").strip() + # Profile-scoped read (adapter startup, Slack pattern + # #59739): a scoped read honors the profile's own secret; + # only an UNSCOPED read under multiplex (default-profile + # startup loop) falls back to the process env, which is that + # profile's own value. + from agent.secret_scope import ( + UnscopedSecretError, + get_secret, + ) + + try: + webhook_secret = (get_secret("TELEGRAM_WEBHOOK_SECRET") or "").strip() + except UnscopedSecretError: + webhook_secret = os.getenv("TELEGRAM_WEBHOOK_SECRET", "").strip() if not webhook_secret: raise RuntimeError( "TELEGRAM_WEBHOOK_SECRET is required when " @@ -9932,7 +9945,13 @@ async def _standalone_send( parse-mode fallback). Implements the standalone_sender_fn contract so deliver=telegram cron jobs succeed when cron runs separately from the gateway.""" - token = getattr(pconfig, "token", None) or os.getenv("TELEGRAM_BOT_TOKEN", "") + token = getattr(pconfig, "token", None) + if not token: + # Profile-scoped read: honor the secret scope's verdict rather than + # borrowing another profile's env-bridged token under multiplex. + from agent.secret_scope import get_secret + + token = get_secret("TELEGRAM_BOT_TOKEN", "") or "" disable_link_previews = bool( getattr(pconfig, "extra", {}) and pconfig.extra.get("disable_link_previews") ) From 5c6cc38010478a0604bc304a1d25884aed18e1f4 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 2 Aug 2026 00:52:55 -0700 Subject: [PATCH 34/69] fix(secrets): scope-aware credential reads in core tool/gateway/web-server paths TOOL_GATEWAY_USER_TOKEN (managed_tool_gateway), OPENROUTER_API_KEY presence (openrouter_client), SUDO_PASSWORD (terminal_tool), GATEWAY_PROXY_KEY (gateway/run), SLACK_BOT_TOKEN presence (gateway/session), and the ELEVENLABS_API_KEY env fallback (web_server voices endpoint) now honor the installed profile secret scope; unscoped callers keep legacy env reads via the UnscopedSecretError fallback (Slack pattern). --- gateway/run.py | 13 ++++++++++++- gateway/session.py | 14 +++++++++++++- hermes_cli/web_server.py | 15 ++++++++++++++- tools/managed_tool_gateway.py | 34 ++++++++++++++++++++++++++++------ tools/openrouter_client.py | 16 +++++++++++++++- tools/terminal_tool.py | 16 ++++++++++++++-- 6 files changed, 96 insertions(+), 12 deletions(-) diff --git a/gateway/run.py b/gateway/run.py index 6c97d02260ba..2d870b2cd8cf 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -23074,7 +23074,18 @@ async def _run_agent_via_proxy( "tools": [], } - proxy_key = os.getenv("GATEWAY_PROXY_KEY", "").strip() + # Scope-aware read: the proxy key is a per-profile credential; under + # multiplex honor the installed scope's verdict (Slack pattern for + # the unscoped default-profile loop). + try: + from agent.secret_scope import UnscopedSecretError, get_secret + + try: + proxy_key = (get_secret("GATEWAY_PROXY_KEY") or "").strip() + except UnscopedSecretError: + proxy_key = os.getenv("GATEWAY_PROXY_KEY", "").strip() + except Exception: + proxy_key = os.getenv("GATEWAY_PROXY_KEY", "").strip() def _run_still_current() -> bool: if run_generation is None or not session_key: diff --git a/gateway/session.py b/gateway/session.py index 2d5cebd32026..4335ad54bcaf 100644 --- a/gateway/session.py +++ b/gateway/session.py @@ -387,7 +387,19 @@ def _slack_tools_loaded() -> bool: except Exception: pass - if not (os.environ.get("SLACK_BOT_TOKEN") or "").strip(): + # Presence check through the profile secret scope: under multiplex the + # process env may carry another profile's token (Slack pattern for the + # unscoped default-profile path). + try: + from agent.secret_scope import UnscopedSecretError, get_secret + + try: + _slack_token = get_secret("SLACK_BOT_TOKEN") or "" + except UnscopedSecretError: + _slack_token = os.environ.get("SLACK_BOT_TOKEN") or "" + except Exception: + _slack_token = os.environ.get("SLACK_BOT_TOKEN") or "" + if not _slack_token.strip(): return False try: from hermes_cli.config import load_config diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index 2f4ace574b10..9e7bf8421abe 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -4320,7 +4320,20 @@ async def get_elevenlabs_voices(profile: Optional[str] = None): # Config-only scope (await-safe): the key lookup reads the requested # profile's .env, matching the profile the settings UI writes to. with _config_profile_scope(profile): - api_key = (load_env().get("ELEVENLABS_API_KEY") or os.environ.get("ELEVENLABS_API_KEY") or "").strip() + api_key = (load_env().get("ELEVENLABS_API_KEY") or "").strip() + if not api_key: + # Fallback for env-only deployments — scope-aware (Slack pattern): + # under multiplex os.environ may hold another profile's key, so + # honor the installed scope's verdict before touching the env. + try: + from agent.secret_scope import UnscopedSecretError, get_secret + + try: + api_key = (get_secret("ELEVENLABS_API_KEY") or "").strip() + except UnscopedSecretError: + api_key = (os.environ.get("ELEVENLABS_API_KEY") or "").strip() + except Exception: + api_key = (os.environ.get("ELEVENLABS_API_KEY") or "").strip() if not api_key: return {"available": False, "voices": []} diff --git a/tools/managed_tool_gateway.py b/tools/managed_tool_gateway.py index af7f8f69748d..c46a48975bb4 100644 --- a/tools/managed_tool_gateway.py +++ b/tools/managed_tool_gateway.py @@ -73,6 +73,28 @@ def _access_token_is_expiring(expires_at: object, skew_seconds: int) -> bool: return remaining <= max(0, int(skew_seconds)) +def _read_user_token_override() -> Optional[str]: + """Read the TOOL_GATEWAY_USER_TOKEN env override through the secret scope. + + Availability scans run both inside agent turns (scope installed) and in + unscoped CLI paths, so this uses the Slack pattern: honor the scope's + verdict when installed (a scoped miss does NOT borrow the process env + under multiplex), fall back to ``os.environ`` only when unscoped. + """ + try: + from agent.secret_scope import UnscopedSecretError, get_secret + + try: + explicit = get_secret("TOOL_GATEWAY_USER_TOKEN") + except UnscopedSecretError: + explicit = os.getenv("TOOL_GATEWAY_USER_TOKEN") + except Exception: + explicit = os.getenv("TOOL_GATEWAY_USER_TOKEN") + if isinstance(explicit, str) and explicit.strip(): + return explicit.strip() + return None + + def peek_nous_access_token() -> Optional[str]: """Cheap probe for a Nous gateway token without triggering refresh. @@ -83,9 +105,9 @@ def peek_nous_access_token() -> Optional[str]: network calls. Truthful refresh handling stays in request/session paths that call :func:`read_nous_access_token`. """ - explicit = os.getenv("TOOL_GATEWAY_USER_TOKEN") - if isinstance(explicit, str) and explicit.strip(): - return explicit.strip() + explicit = _read_user_token_override() + if explicit: + return explicit nous_provider = _read_nous_provider_state() or {} access_token = nous_provider.get("access_token") @@ -96,9 +118,9 @@ def peek_nous_access_token() -> Optional[str]: def read_nous_access_token() -> Optional[str]: """Read a Nous Subscriber OAuth access token from auth store or env override.""" - explicit = os.getenv("TOOL_GATEWAY_USER_TOKEN") - if isinstance(explicit, str) and explicit.strip(): - return explicit.strip() + explicit = _read_user_token_override() + if explicit: + return explicit nous_provider = _read_nous_provider_state() or {} cached_token = peek_nous_access_token() diff --git a/tools/openrouter_client.py b/tools/openrouter_client.py index 0637a7db0ded..9c857076c875 100644 --- a/tools/openrouter_client.py +++ b/tools/openrouter_client.py @@ -29,5 +29,19 @@ def get_async_client(): def check_api_key() -> bool: - """Check whether the OpenRouter API key is present.""" + """Check whether the OpenRouter API key is present. + + Scope-aware (Slack pattern): tool paths run inside an installed profile + secret scope, whose verdict is authoritative under multiplex; unscoped + CLI probes keep the legacy env read. + """ + try: + from agent.secret_scope import UnscopedSecretError, get_secret + + try: + return bool(get_secret("OPENROUTER_API_KEY")) + except UnscopedSecretError: + pass + except Exception: + pass return bool(os.getenv("OPENROUTER_API_KEY")) diff --git a/tools/terminal_tool.py b/tools/terminal_tool.py index 7f9141ff8a9e..244449ce97bc 100644 --- a/tools/terminal_tool.py +++ b/tools/terminal_tool.py @@ -977,9 +977,21 @@ def _transform_sudo_command(command: str | None) -> tuple[str | None, str | None if sudo_count == 0: return command, None - has_configured_password = "SUDO_PASSWORD" in os.environ + # Scope-aware read (Slack pattern): under multiplex the process env may + # hold another profile's SUDO_PASSWORD, so honor the installed scope's + # verdict; unscoped callers keep the legacy os.environ read. + try: + from agent.secret_scope import UnscopedSecretError, get_secret + + try: + _configured_password = get_secret("SUDO_PASSWORD") + except UnscopedSecretError: + _configured_password = os.environ.get("SUDO_PASSWORD") + except Exception: + _configured_password = os.environ.get("SUDO_PASSWORD") + has_configured_password = _configured_password is not None sudo_password = ( - os.environ.get("SUDO_PASSWORD", "") + _configured_password if has_configured_password else _get_cached_sudo_password() ) From ca5ce1110b80e420876188a75c9e8f671566bd67 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 2 Aug 2026 00:54:06 -0700 Subject: [PATCH 35/69] fix(secrets): route auxiliary-client provider key reads through the profile secret scope MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OPENROUTER_API_KEY (client construction, unavailability description, max-tokens param heuristic), OPENAI_API_KEY (custom runtime + explicit custom endpoint), named-custom-provider key_env, and auxiliary task-config key_env now resolve via a shared _scoped_key_env helper (get_secret with UnscopedSecretError → os.environ fallback for unscoped CLI/startup probes). --- agent/auxiliary_client.py | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index 980813f12358..22bf9c165cf0 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -1133,6 +1133,29 @@ def _nous_min_key_ttl_seconds() -> int: return 1800 +def _scoped_key_env(name: str) -> str: + """Read a provider API key env var through the profile secret scope. + + Auxiliary-client resolution runs both inside agent turns (secret scope + installed — its verdict is authoritative under multiplex, so a scoped + miss must NOT borrow another profile's process-env key) and on unscoped + startup/CLI probe paths, which keep the legacy ``os.environ`` read via + the ``UnscopedSecretError`` fallback (Slack pattern, #59739). + """ + if not name: + return "" + try: + from agent.secret_scope import UnscopedSecretError, get_secret + + try: + return (get_secret(name) or "").strip() + except UnscopedSecretError: + pass + except Exception: + pass + return (os.getenv(name) or "").strip() + + # ── Codex Responses → chat.completions adapter ───────────────────────────── # All auxiliary consumers call client.chat.completions.create(**kwargs) and # read response.choices[0].message.content. This adapter translates those @@ -2470,7 +2493,7 @@ def _try_openrouter(explicit_api_key: str = None, model: str = None) -> Tuple[Op # the OPENROUTER_API_KEY env-var path rather than failing outright. logger.debug("Auxiliary client: OpenRouter pool exhausted, trying OPENROUTER_API_KEY") - or_key = explicit_api_key or os.getenv("OPENROUTER_API_KEY") + or_key = explicit_api_key or _scoped_key_env("OPENROUTER_API_KEY") if not or_key: _mark_provider_unhealthy("openrouter", ttl=60) return None, None @@ -2487,7 +2510,7 @@ def _describe_openrouter_unavailable() -> str: return "OpenRouter credential pool has no usable entries (credentials may be exhausted)" if not _pool_runtime_api_key(entry): return "OpenRouter credential pool entry is missing a runtime API key" - if not str(os.getenv("OPENROUTER_API_KEY") or "").strip(): + if not _scoped_key_env("OPENROUTER_API_KEY"): return "OPENROUTER_API_KEY not set" return "no usable OpenRouter credentials found" @@ -3119,7 +3142,7 @@ def _resolve_custom_runtime() -> Tuple[Optional[str], Optional[str], Optional[st if not isinstance(runtime, dict): openai_base = os.getenv("OPENAI_BASE_URL", "").strip().rstrip("/") - openai_key = os.getenv("OPENAI_API_KEY", "").strip() + openai_key = _scoped_key_env("OPENAI_API_KEY") if not openai_base: return None, None, None runtime = { @@ -5925,7 +5948,7 @@ def _wrap_if_needed(client_obj, final_model_str: str, base_url_str: str = "", custom_base = _to_openai_base_url(explicit_base_url).strip() custom_key = ( (explicit_api_key or "").strip() - or os.getenv("OPENAI_API_KEY", "").strip() + or _scoped_key_env("OPENAI_API_KEY") or _read_main_api_key_if_same_host(custom_base) or "no-key-required" # local servers don't need auth ) @@ -6021,7 +6044,7 @@ def _wrap_if_needed(client_obj, final_model_str: str, base_url_str: str = "", custom_key = (custom_entry.get("api_key") or "").strip() custom_key_env = (custom_entry.get("key_env") or custom_entry.get("api_key_env") or "").strip() if not custom_key and custom_key_env: - custom_key = os.getenv(custom_key_env, "").strip() + custom_key = _scoped_key_env(custom_key_env) custom_key = custom_key or "no-key-required" if custom_key == "no-key-required": logger.warning( @@ -6843,7 +6866,7 @@ def auxiliary_max_tokens_param(value: int, *, model: Optional[str] = None) -> di misses the case where a custom base URL serves e.g. ``gpt-5.4``. """ custom_base = _current_custom_base_url() - or_key = os.getenv("OPENROUTER_API_KEY") + or_key = _scoped_key_env("OPENROUTER_API_KEY") # Use max_completion_tokens for direct OpenAI-compatible providers that reject # max_tokens on newer GPT-4o/o-series/GPT-5-style models. _custom_host = base_url_hostname(custom_base) or "" @@ -7304,7 +7327,7 @@ def _resolve_task_provider_model( task_config.get("key_env") or task_config.get("api_key_env") or "" ).strip() if cfg_key_env: - cfg_api_key = os.getenv(cfg_key_env, "").strip() or None + cfg_api_key = _scoped_key_env(cfg_key_env) or None cfg_api_mode = str(task_config.get("api_mode", "")).strip() or None # 'auto' is a sentinel meaning "inherit from main runtime / auto-detect", not From 5239b4d6d58577f8e4262f337576f5653b7a2464 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 2 Aug 2026 00:54:21 -0700 Subject: [PATCH 36/69] fix(secrets): scope-aware Azure credential presence reads in identity diagnostics AZURE_CLIENT_SECRET and AZURE_FEDERATED_TOKEN_FILE presence checks in the Entra diagnostics path now read through the profile secret scope (Slack pattern) instead of raw os.environ. --- agent/azure_identity_adapter.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/agent/azure_identity_adapter.py b/agent/azure_identity_adapter.py index 9506715019d7..dd0f62ab9737 100644 --- a/agent/azure_identity_adapter.py +++ b/agent/azure_identity_adapter.py @@ -367,11 +367,27 @@ class name. Users wanting the precise class can run with info["tenant_id_env"] = os.environ["AZURE_TENANT_ID"].strip() # Surface which env-var sources are present without minting yet. + # Credential-bearing vars (AZURE_CLIENT_SECRET, AZURE_FEDERATED_TOKEN_FILE) + # are read through the profile secret scope so a multiplexed profile's + # diagnostics don't report another profile's env-bridged credentials; + # unscoped CLI probes keep the legacy env read (Slack pattern). + def _scoped_env(name: str) -> str: + try: + from agent.secret_scope import UnscopedSecretError, get_secret + + try: + return (get_secret(name) or "").strip() + except UnscopedSecretError: + pass + except Exception: + pass + return os.environ.get(name, "").strip() + env_sources = [] - if os.environ.get("AZURE_FEDERATED_TOKEN_FILE", "").strip(): + if _scoped_env("AZURE_FEDERATED_TOKEN_FILE"): env_sources.append("WorkloadIdentityCredential (AZURE_FEDERATED_TOKEN_FILE)") if (os.environ.get("AZURE_CLIENT_ID", "").strip() - and os.environ.get("AZURE_CLIENT_SECRET", "").strip() + and _scoped_env("AZURE_CLIENT_SECRET") and os.environ.get("AZURE_TENANT_ID", "").strip()): env_sources.append("EnvironmentCredential (client secret)") if os.environ.get("IDENTITY_ENDPOINT", "").strip() or os.environ.get("MSI_ENDPOINT", "").strip(): From 2c9bbda072d98e828788f4e27d175a14a11ffdd8 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 2 Aug 2026 00:56:09 -0700 Subject: [PATCH 37/69] test(secrets): consolidated tier-1 migration regression suite Exercises representative migrated sites per cluster: scoped value wins, scoped miss no-borrow under multiplex, and unscoped-multiplex behavior per pattern (in-turn get_secret vs Slack-pattern startup fallback). --- .../test_secret_scope_tier1_migration.py | 233 ++++++++++++++++++ 1 file changed, 233 insertions(+) create mode 100644 tests/agent/test_secret_scope_tier1_migration.py diff --git a/tests/agent/test_secret_scope_tier1_migration.py b/tests/agent/test_secret_scope_tier1_migration.py new file mode 100644 index 000000000000..cceb32e21f61 --- /dev/null +++ b/tests/agent/test_secret_scope_tier1_migration.py @@ -0,0 +1,233 @@ +"""Regression tests for the Tier-1 core-gateway secret-scope migration. + +Class-closure follow-up to the profile secret-scope cluster (#76462 / +#76574): representative call sites from each migrated cluster are exercised +against the three canonical scope semantics: + +- scoped value wins (the installed profile's secret is used), +- scoped miss does NOT borrow the process env under multiplex (no-borrow), +- unscoped-under-multiplex behavior per pattern: + * in-turn sites (get_secret direct) propagate/honor UnscopedSecretError + semantics via get_secret's verdict, + * startup sites (Slack pattern) fall back to os.environ on + UnscopedSecretError. +""" + +import pytest + +from agent import secret_scope as ss + + +@pytest.fixture(autouse=True) +def _reset_multiplex(): + ss.set_multiplex_active(False) + yield + ss.set_multiplex_active(False) + + +class _Scope: + """Context manager installing a secret scope.""" + + def __init__(self, mapping): + self.mapping = mapping + self.token = None + + def __enter__(self): + self.token = ss.set_secret_scope(self.mapping) + return self + + def __exit__(self, *exc): + ss.reset_secret_scope(self.token) + + +# ── Cluster A: gateway/pairing.py allowlist reads ───────────────────────── + +class TestPairingAllowlistRead: + def test_scoped_value_wins(self, monkeypatch): + from gateway.pairing import _read_allowlist_env + + monkeypatch.setenv("TELEGRAM_ALLOWED_USERS", "111") + ss.set_multiplex_active(True) + with _Scope({"TELEGRAM_ALLOWED_USERS": "222"}): + assert _read_allowlist_env("TELEGRAM_ALLOWED_USERS") == "222" + + def test_scoped_miss_no_borrow(self, monkeypatch): + from gateway.pairing import _read_allowlist_env + + monkeypatch.setenv("TELEGRAM_ALLOWED_USERS", "other-profile") + ss.set_multiplex_active(True) + with _Scope({"UNRELATED": "x"}): + assert _read_allowlist_env("TELEGRAM_ALLOWED_USERS") == "" + + def test_unscoped_multiplex_falls_back_to_env(self, monkeypatch): + # Slack pattern: unscoped read under multiplex uses the process env. + from gateway.pairing import _read_allowlist_env + + monkeypatch.setenv("TELEGRAM_ALLOWED_USERS", "own-env") + ss.set_multiplex_active(True) + assert _read_allowlist_env("TELEGRAM_ALLOWED_USERS") == "own-env" + + +# ── Cluster A: gateway/authz_mixin.py gate reads ─────────────────────────── + +class TestAuthzPlatformGateEnv: + def test_scoped_value_wins(self, monkeypatch): + from gateway.authz_mixin import _platform_gate_env + + monkeypatch.setenv("DISCORD_ALLOW_BOTS", "none") + ss.set_multiplex_active(True) + with _Scope({"DISCORD_ALLOW_BOTS": "all"}): + assert _platform_gate_env("DISCORD_ALLOW_BOTS", "none") == "all" + + def test_scoped_miss_returns_default_not_env(self, monkeypatch): + from gateway.authz_mixin import _platform_gate_env + + monkeypatch.setenv("DISCORD_ALLOW_BOTS", "all") # another profile's bridge + ss.set_multiplex_active(True) + with _Scope({"UNRELATED": "x"}): + assert _platform_gate_env("DISCORD_ALLOW_BOTS", "none") == "none" + + def test_single_profile_legacy_env(self, monkeypatch): + from gateway.authz_mixin import _platform_gate_env + + monkeypatch.setenv("GATEWAY_ALLOWED_USERS", "42") + assert _platform_gate_env("GATEWAY_ALLOWED_USERS") == "42" + + +# ── Cluster B: matrix startup reads (Slack pattern) ──────────────────────── + +class TestMatrixStartupSecret: + def _helper(self): + mod = pytest.importorskip("plugins.platforms.matrix.adapter") + return mod._startup_env_secret + + def test_scoped_value_wins(self, monkeypatch): + helper = self._helper() + monkeypatch.setenv("MATRIX_ACCESS_TOKEN", "env-token") + ss.set_multiplex_active(True) + with _Scope({"MATRIX_ACCESS_TOKEN": "scoped-token"}): + assert helper("MATRIX_ACCESS_TOKEN") == "scoped-token" + + def test_scoped_miss_no_borrow(self, monkeypatch): + helper = self._helper() + monkeypatch.setenv("MATRIX_ACCESS_TOKEN", "other-profile") + ss.set_multiplex_active(True) + with _Scope({"UNRELATED": "x"}): + assert helper("MATRIX_ACCESS_TOKEN") == "" + + def test_unscoped_multiplex_falls_back(self, monkeypatch): + helper = self._helper() + monkeypatch.setenv("MATRIX_PASSWORD", "own-env-pass") + ss.set_multiplex_active(True) + assert helper("MATRIX_PASSWORD") == "own-env-pass" + + +# ── Cluster C: managed tool gateway token override ───────────────────────── + +class TestToolGatewayUserToken: + def test_scoped_value_wins(self, monkeypatch): + from tools.managed_tool_gateway import _read_user_token_override + + monkeypatch.setenv("TOOL_GATEWAY_USER_TOKEN", "env-tok") + ss.set_multiplex_active(True) + with _Scope({"TOOL_GATEWAY_USER_TOKEN": "scoped-tok"}): + assert _read_user_token_override() == "scoped-tok" + + def test_scoped_miss_no_borrow(self, monkeypatch): + from tools.managed_tool_gateway import _read_user_token_override + + monkeypatch.setenv("TOOL_GATEWAY_USER_TOKEN", "other-profile-tok") + ss.set_multiplex_active(True) + with _Scope({"UNRELATED": "x"}): + assert _read_user_token_override() is None + + def test_unscoped_multiplex_falls_back(self, monkeypatch): + from tools.managed_tool_gateway import _read_user_token_override + + monkeypatch.setenv("TOOL_GATEWAY_USER_TOKEN", "own-env-tok") + ss.set_multiplex_active(True) + assert _read_user_token_override() == "own-env-tok" + + +class TestOpenRouterCheckApiKey: + def test_scoped_value_wins(self, monkeypatch): + from tools.openrouter_client import check_api_key + + monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) + ss.set_multiplex_active(True) + with _Scope({"OPENROUTER_API_KEY": "sk-or-scoped"}): + assert check_api_key() is True + + def test_scoped_miss_no_borrow(self, monkeypatch): + from tools.openrouter_client import check_api_key + + monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-other-profile") + ss.set_multiplex_active(True) + with _Scope({"UNRELATED": "x"}): + assert check_api_key() is False + + +# ── Cluster D: auxiliary client key resolution ────────────────────────────── + +class TestAuxiliaryScopedKeyEnv: + def test_scoped_value_wins(self, monkeypatch): + from agent.auxiliary_client import _scoped_key_env + + monkeypatch.setenv("OPENROUTER_API_KEY", "sk-env") + ss.set_multiplex_active(True) + with _Scope({"OPENROUTER_API_KEY": "sk-scoped"}): + assert _scoped_key_env("OPENROUTER_API_KEY") == "sk-scoped" + + def test_scoped_miss_no_borrow(self, monkeypatch): + from agent.auxiliary_client import _scoped_key_env + + monkeypatch.setenv("OPENAI_API_KEY", "sk-other-profile") + ss.set_multiplex_active(True) + with _Scope({"UNRELATED": "x"}): + assert _scoped_key_env("OPENAI_API_KEY") == "" + + def test_unscoped_multiplex_falls_back(self, monkeypatch): + from agent.auxiliary_client import _scoped_key_env + + monkeypatch.setenv("OPENAI_API_KEY", "sk-own-env") + ss.set_multiplex_active(True) + assert _scoped_key_env("OPENAI_API_KEY") == "sk-own-env" + + def test_empty_name_returns_empty(self): + from agent.auxiliary_client import _scoped_key_env + + assert _scoped_key_env("") == "" + + +# ── Cluster E: azure identity presence reads ──────────────────────────────── + +class TestAzureIdentityPresence: + def _describe(self): + azure = pytest.importorskip("agent.azure_identity_adapter") + if not azure.has_azure_identity_installed(): + pytest.skip("azure-identity not installed") + return azure.describe_active_credential + + def test_scoped_client_secret_detected(self, monkeypatch): + describe = self._describe() + monkeypatch.setenv("AZURE_CLIENT_ID", "cid") + monkeypatch.setenv("AZURE_TENANT_ID", "tid") + monkeypatch.delenv("AZURE_CLIENT_SECRET", raising=False) + monkeypatch.delenv("AZURE_FEDERATED_TOKEN_FILE", raising=False) + ss.set_multiplex_active(True) + with _Scope({"AZURE_CLIENT_SECRET": "scoped-secret"}): + info = describe(timeout_seconds=0.01, allow_install=False) + assert any("EnvironmentCredential" in s for s in info.get("env_sources", [])) + + def test_scoped_miss_hides_env_secret(self, monkeypatch): + describe = self._describe() + monkeypatch.setenv("AZURE_CLIENT_ID", "cid") + monkeypatch.setenv("AZURE_TENANT_ID", "tid") + monkeypatch.setenv("AZURE_CLIENT_SECRET", "other-profile-secret") + monkeypatch.delenv("AZURE_FEDERATED_TOKEN_FILE", raising=False) + ss.set_multiplex_active(True) + with _Scope({"UNRELATED": "x"}): + info = describe(timeout_seconds=0.01, allow_install=False) + assert not any( + "EnvironmentCredential" in s for s in info.get("env_sources", []) + ) From 50c4afe40ad09c1650756f1017b28b733c9cd4a4 Mon Sep 17 00:00:00 2001 From: Soju06 Date: Tue, 14 Jul 2026 04:01:00 +0000 Subject: [PATCH 38/69] perf(gateway): single-row routing UPSERT fast path for metadata-only saves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The steady-state turn only bumps updated_at/last_prompt_tokens on one routing entry, but persisted it through the full index rewrite twice per turn (get_or_create_session's healthy-path bump + update_session): every entry re-serialized, DELETE+INSERT of every gateway_routing row, and a multi-MB sessions.json dump+fsync — ~50ms p50 at ~1,100 routing keys in production, out of ~175ms total per-turn gateway persistence. Metadata-only saves now UPSERT the single row via the existing HermesDB.save_gateway_routing_entry (<1ms). Structural transitions (create/recover/reset/switch/prune, compression-tip heals) keep the full rewrite, which also refreshes the legacy sessions.json mirror. Correctness: each fast save allocates a per-entry revision from the routing generation counter under _lock, so fast and full snapshots are totally ordered by number. Under _save_lock the UPSERT is skipped when a newer full snapshot or a newer fast save of the same key has already persisted, and a delayed full rewrite folds in fast records serialized after its snapshot before writing — an older snapshot can never overwrite a newer one, in either direction. update_session snapshots peer fields under _lock so a concurrent reset cannot record a torn peer row; no DB or a failed UPSERT falls back to the full rewrite so DB-less installs keep sessions.json durable every turn. --- gateway/session.py | 154 ++++++- tests/gateway/test_routing_save_fast_path.py | 445 +++++++++++++++++++ 2 files changed, 584 insertions(+), 15 deletions(-) create mode 100644 tests/gateway/test_routing_save_fast_path.py diff --git a/gateway/session.py b/gateway/session.py index 4335ad54bcaf..c99b8a39cd96 100644 --- a/gateway/session.py +++ b/gateway/session.py @@ -1224,6 +1224,11 @@ def __init__(self, sessions_dir: Path, config: GatewayConfig, self._save_lock = threading.Lock() self._routing_generation = 0 self._persisted_routing_generation = 0 + # Single-entry upserts persisted since the last full rewrite: + # session_key -> (revision, entry_json). Revisions are allocated + # from _routing_generation, so fast and full snapshots are totally + # ordered; guarded by _save_lock (see _save_entry). + self._fast_persisted_entries: Dict[str, tuple[int, str]] = {} self._inflight_lock = threading.Lock() self._inflight_sessions: Dict[str, _SessionFlight] = {} # An unscoped pre-migration Slack key can represent at most one @@ -1488,6 +1493,16 @@ def _persist_routing_data(self, data: Dict[str, Any], generation: int) -> None: with save_lock: if generation <= getattr(self, "_persisted_routing_generation", 0): return + # Fold in single-entry upserts with a newer revision than this + # snapshot (see _save_entry): revisions share the routing + # generation counter, so a fast record numbered above us was + # serialized after us and a delayed full rewrite must not + # regress it. + fast_persisted = getattr(self, "_fast_persisted_entries", None) + if fast_persisted: + for key, (revision, entry_json) in fast_persisted.items(): + if revision > generation: + data[key] = json.loads(entry_json) db_saved = False _db = getattr(self, "_db", None) if _db: @@ -1506,6 +1521,14 @@ def _persist_routing_data(self, data: Dict[str, Any], generation: int) -> None: if getattr(self, "_write_sessions_json", True) or not db_saved: self._save_sessions_json(data) self._persisted_routing_generation = generation + # This rewrite supersedes fast records at or below its + # generation; newer ones stay for the next delayed full writer. + if fast_persisted: + for key in [ + k for k, (rev, _) in fast_persisted.items() + if rev <= generation + ]: + del fast_persisted[key] def _save_sessions_json(self, data: Dict[str, Any]) -> None: """Write the legacy sessions.json mirror of the routing index.""" @@ -1552,6 +1575,85 @@ def _save_entries(self) -> None: with self._lock: data, generation = self._snapshot_routing_locked() self._persist_routing_data(data, generation) + + def _save_entry(self, session_key: str) -> None: + """Persist ONE routing entry via UPSERT — the per-turn fast path. + + The steady-state turn only bumps ``updated_at`` / + ``last_prompt_tokens`` on one entry; routing that through the + full index rewrite re-serializes every entry, DELETE+INSERTs + every gateway_routing row, and dumps+fsyncs a multi-MB + sessions.json — ~50ms p50 at ~1100 routing keys, and it runs + twice per turn. A single-row UPSERT keeps the durable state.db + mapping current in well under a millisecond. + + Correctness constraints this path relies on: + + - The key -> session_id mapping never changes here. Structural + transitions (create/recover/reset/switch/prune, and + compression-tip heals — see get_or_create_session) still use + the full-rewrite path, which also refreshes the legacy + sessions.json mirror. Between structural saves the mirror may + lag in metadata only; every remaining sessions.json reader is + a legacy fallback and state.db stays primary, so restart + rebinding is unaffected. + + - Ordering vs concurrent writers: the entry is serialized under + ``_lock`` together with a revision allocated from the routing + generation counter, so every snapshot — fast or full — carries + a unique, monotonically increasing number, and a higher number + always means same-or-newer data for this key. Under + ``_save_lock`` the upsert is skipped when a snapshot numbered + above ours already persisted this key: a FULL snapshot + (``_persisted_routing_generation``) or another fast save of + the same key (``_fast_persisted_entries``). Either contains a + same-or-newer copy, so writing ours would regress it. The + reverse interaction — a delayed full rewrite landing after a + later-serialized fast save — is handled in + ``_persist_routing_data``, which folds fast records numbered + above its snapshot into the rewrite. An older snapshot can + therefore never overwrite a newer one, in either direction. + + - No DB, or a failed upsert, falls back to the full rewrite so + DB-less installs keep sessions.json — their primary store — + durable every turn. + """ + with self._lock: + entry = self._entries.get(session_key) + if entry is None: + return + entry_json = json.dumps(entry.to_dict()) + self._routing_generation = getattr(self, "_routing_generation", 0) + 1 + revision = self._routing_generation + _db = getattr(self, "_db", None) + saver = getattr(_db, "save_gateway_routing_entry", None) if _db else None + if callable(saver): + save_lock = getattr(self, "_save_lock", None) + if save_lock is None: + save_lock = threading.Lock() + self._save_lock = save_lock + try: + with save_lock: + if getattr(self, "_persisted_routing_generation", 0) >= revision: + return + fast_persisted = getattr(self, "_fast_persisted_entries", None) + if fast_persisted is None: + fast_persisted = {} + self._fast_persisted_entries = fast_persisted + persisted = fast_persisted.get(session_key) + if persisted is not None and persisted[0] >= revision: + return + saver(session_key, entry_json, scope=self._routing_scope()) + fast_persisted[session_key] = (revision, entry_json) + return + except Exception as exc: + logger.warning( + "gateway.session: single-entry routing save failed for %r " + "(%s); falling back to full index rewrite", + session_key, exc, + ) + self._save_entries() + def _resolve_profile_for_key(self, source: Optional[SessionSource] = None) -> Optional[str]: """Return the profile namespace for session keys, or None when off. @@ -2346,6 +2448,11 @@ def _get_or_create_session_impl( # ---- Phase 2: lock write -- apply decisions to _entries ---- _needs_save = False + # Healthy-path saves only bump updated_at on one entry; they take + # the single-row UPSERT fast path instead of the full index rewrite + # (see _save_entry). Structural transitions (recover/create below) + # keep the full rewrite. + _metadata_only_save = False _needs_recover = False entry: Optional[SessionEntry] = None was_auto_reset = False @@ -2358,7 +2465,10 @@ def _get_or_create_session_impl( if session_key in self._entries and not force_new: entry = self._entries[session_key] - self._heal_compression_tip_locked( + # A heal rewrites entry.session_id, so it must reach the + # sessions.json mirror too: force the full-rewrite save + # below (the fast path persists state.db only). + _healed = self._heal_compression_tip_locked( entry, existing_session_id, canonical_existing_session_id ) @@ -2394,6 +2504,7 @@ def _get_or_create_session_impl( # window. Treat as healthy -- bump updated_at and save. entry.updated_at = now _needs_save = True + _metadata_only_save = not _healed else: # Stale check clean. Apply reset decision. if _reset_reason: @@ -2408,6 +2519,7 @@ def _get_or_create_session_impl( else: entry.updated_at = now _needs_save = True + _metadata_only_save = not _healed else: if not force_new: _needs_recover = True @@ -2474,7 +2586,10 @@ def _get_or_create_session_impl( } if _needs_save: - self._save_entries() + if _metadata_only_save: + self._save_entry(session_key) + else: + self._save_entries() # SQLite operations outside the lock (unchanged). if self._db and db_end_session_id: @@ -2518,19 +2633,28 @@ def update_session( """Update lightweight session metadata after an interaction.""" with self._lock: self._ensure_loaded_locked() - - if session_key in self._entries: - entry = self._entries[session_key] - entry.updated_at = _now() - if last_prompt_tokens is not None: - entry.last_prompt_tokens = last_prompt_tokens - self._save() - self._record_gateway_session_peer( - entry.session_id, - session_key, - entry.origin, - display_name=entry.display_name, - ) + entry = self._entries.get(session_key) + if entry is None: + return + entry.updated_at = _now() + if last_prompt_tokens is not None: + entry.last_prompt_tokens = last_prompt_tokens + # Snapshot peer fields while still holding _lock: a concurrent + # reset/heal may rewrite the entry, and mixing old and new + # fields would record a torn peer row. + peer_session_id = entry.session_id + peer_origin = entry.origin + peer_display_name = entry.display_name + # Metadata-only change on one entry: single-row UPSERT instead of + # the full index rewrite (see _save_entry). Both writes run outside + # ``_lock`` so the SQLite commit never blocks routing lookups. + self._save_entry(session_key) + self._record_gateway_session_peer( + peer_session_id, + session_key, + peer_origin, + display_name=peer_display_name, + ) def get_session_metadata( self, diff --git a/tests/gateway/test_routing_save_fast_path.py b/tests/gateway/test_routing_save_fast_path.py new file mode 100644 index 000000000000..d693c3b236d4 --- /dev/null +++ b/tests/gateway/test_routing_save_fast_path.py @@ -0,0 +1,445 @@ +"""Single-row routing save fast path. + +Metadata-only per-turn writes (get_or_create_session's healthy-path +``updated_at`` bump, ``update_session``) persist through a single-row +UPSERT instead of rewriting the whole routing index. These tests prove +the write-skipping never loses a NEEDED write: changed values always +land in state.db, and restart rebinding works even when the legacy +sessions.json mirror lagged behind (or never existed). +""" +from __future__ import annotations + +import json +import threading + +import hermes_state +from gateway.config import GatewayConfig, Platform +from gateway.session import SessionSource, SessionStore + + +def _source(user_id: str = "user-1") -> SessionSource: + return SessionSource( + platform=Platform.LOCAL, + chat_id="cli", + chat_name="CLI", + chat_type="dm", + user_id=user_id, + ) + + +def _make_store(tmp_path, monkeypatch, **config_kwargs) -> SessionStore: + monkeypatch.setattr(hermes_state, "DEFAULT_DB_PATH", tmp_path / "state.db") + return SessionStore( + sessions_dir=tmp_path / "sessions", + config=GatewayConfig(**config_kwargs), + ) + + +def _routing_row(store: SessionStore, session_key: str) -> dict: + rows = store._db.load_gateway_routing_entries(scope=store._routing_scope()) + return json.loads(rows[session_key]) + + +class TestChangedValuesAlwaysPersist: + def test_update_session_persists_last_prompt_tokens_to_db( + self, tmp_path, monkeypatch + ): + store = _make_store(tmp_path, monkeypatch) + entry = store.get_or_create_session(_source()) + + store.update_session(entry.session_key, last_prompt_tokens=54321) + + durable = _routing_row(store, entry.session_key) + assert durable["last_prompt_tokens"] == 54321 + store._db.close() + + def test_healthy_path_bump_persists_updated_at_to_db( + self, tmp_path, monkeypatch + ): + store = _make_store(tmp_path, monkeypatch) + entry = store.get_or_create_session(_source()) + before = _routing_row(store, entry.session_key)["updated_at"] + + # Second lookup takes the healthy-path metadata-only save. + again = store.get_or_create_session(_source()) + assert again.session_id == entry.session_id + + after = _routing_row(store, entry.session_key)["updated_at"] + assert after >= before + # The row reflects the in-memory bump, not a stale copy. + assert after == again.updated_at.isoformat() + store._db.close() + + def test_fast_path_survives_restart_across_stores( + self, tmp_path, monkeypatch + ): + store = _make_store(tmp_path, monkeypatch) + entry = store.get_or_create_session(_source()) + store.update_session(entry.session_key, last_prompt_tokens=777) + store._db.close() + + restarted = _make_store(tmp_path, monkeypatch) + rebound = restarted.get_or_create_session(_source()) + assert rebound.session_id == entry.session_id + assert rebound.last_prompt_tokens == 777 + restarted._db.close() + + +class TestRestartRebindWithoutMirror: + def test_rebind_works_when_mirror_lagged_fast_path_writes( + self, tmp_path, monkeypatch + ): + """The fast path skips sessions.json; state.db alone must rebind.""" + store = _make_store(tmp_path, monkeypatch) + entry = store.get_or_create_session(_source()) + sessions_json = tmp_path / "sessions" / "sessions.json" + assert sessions_json.exists() # structural save wrote the mirror + + # Remove the mirror, then do fast-path-only writes: they must NOT + # recreate it (proves the fast path ran) and must not need it. + sessions_json.unlink() + store.update_session(entry.session_key, last_prompt_tokens=42) + again = store.get_or_create_session(_source()) + assert again.session_id == entry.session_id + assert not sessions_json.exists() + store._db.close() + + restarted = _make_store(tmp_path, monkeypatch) + rebound = restarted.get_or_create_session(_source()) + assert rebound.session_id == entry.session_id + assert rebound.last_prompt_tokens == 42 + restarted._db.close() + + def test_compression_heal_takes_full_path_and_updates_mirror( + self, tmp_path, monkeypatch + ): + """A heal rewrites session_id, so it must bypass the fast path. + + The fast path persists state.db only; if a heal took it, the + sessions.json mirror would keep the ended parent id until the + next structural save (indefinitely on a healthy key). + """ + store = _make_store(tmp_path, monkeypatch) + entry = store.get_or_create_session(_source()) + healed_id = entry.session_id + "_child" + monkeypatch.setattr( + store, + "_compression_tip_for_session_id", + lambda sid: healed_id if sid == entry.session_id else sid, + ) + + again = store.get_or_create_session(_source()) + assert again.session_id == healed_id + + assert _routing_row(store, entry.session_key)["session_id"] == healed_id + sessions_json = tmp_path / "sessions" / "sessions.json" + data = json.loads(sessions_json.read_text(encoding="utf-8")) + assert data[entry.session_key]["session_id"] == healed_id + store._db.close() + + def test_structural_save_still_rewrites_mirror(self, tmp_path, monkeypatch): + store = _make_store(tmp_path, monkeypatch) + first = store.get_or_create_session(_source()) + fresh = store.get_or_create_session(_source(), force_new=True) + assert fresh.session_id != first.session_id + + sessions_json = tmp_path / "sessions" / "sessions.json" + data = json.loads(sessions_json.read_text(encoding="utf-8")) + assert data[fresh.session_key]["session_id"] == fresh.session_id + store._db.close() + + +class TestFallbacks: + def test_no_db_falls_back_to_full_rewrite(self, tmp_path, monkeypatch): + """DB-less installs keep sessions.json durable every turn.""" + store = _make_store(tmp_path, monkeypatch) + entry = store.get_or_create_session(_source()) + store._db.close() + store._db = None + + store.update_session(entry.session_key, last_prompt_tokens=99) + + sessions_json = tmp_path / "sessions" / "sessions.json" + data = json.loads(sessions_json.read_text(encoding="utf-8")) + assert data[entry.session_key]["last_prompt_tokens"] == 99 + + def test_failed_upsert_falls_back_to_full_rewrite( + self, tmp_path, monkeypatch + ): + store = _make_store(tmp_path, monkeypatch) + entry = store.get_or_create_session(_source()) + + def boom(*args, **kwargs): + raise RuntimeError("disk on fire") + + monkeypatch.setattr(store._db, "save_gateway_routing_entry", boom) + store.update_session(entry.session_key, last_prompt_tokens=1234) + + # The full rewrite carried the change to both stores. + sessions_json = tmp_path / "sessions" / "sessions.json" + data = json.loads(sessions_json.read_text(encoding="utf-8")) + assert data[entry.session_key]["last_prompt_tokens"] == 1234 + assert _routing_row(store, entry.session_key)["last_prompt_tokens"] == 1234 + store._db.close() + + +class TestPeerRecordConsistency: + def test_update_session_records_peer_fields_snapshotted_under_lock( + self, tmp_path, monkeypatch + ): + """Peer fields must come from one lock-held snapshot, not late reads. + + The peer record runs outside ``_lock``; a concurrent reset that + rewrites the entry in that window must not produce a torn + old/new field mix in the recorded row. + """ + store = _make_store(tmp_path, monkeypatch) + entry = store.get_or_create_session(_source()) + original_id = entry.session_id + + recorded = [] + monkeypatch.setattr( + store, + "_record_gateway_session_peer", + lambda sid, key, origin, display_name=None: recorded.append( + (sid, key, display_name) + ), + ) + orig_save_entry = store._save_entry + + def swap_then_save(key): + # Simulate a concurrent reset landing after update_session + # released _lock but before the peer record runs. + with store._lock: + store._entries[key].session_id = "reset-rewrote-me" + orig_save_entry(key) + + monkeypatch.setattr(store, "_save_entry", swap_then_save) + store.update_session(entry.session_key, last_prompt_tokens=5) + + assert recorded == [ + (original_id, entry.session_key, entry.display_name) + ] + store._db.close() + + +class TestGenerationOrdering: + def test_upsert_skipped_when_newer_full_snapshot_persisted( + self, tmp_path, monkeypatch + ): + """A full snapshot taken after our serialize point must win. + + Its copy of the key is same-or-newer, so writing ours would + regress it. Simulated by advancing _persisted_routing_generation + past the generation captured at serialize time. + """ + store = _make_store(tmp_path, monkeypatch) + entry = store.get_or_create_session(_source()) + + calls = [] + monkeypatch.setattr( + store._db, + "save_gateway_routing_entry", + lambda *a, **k: calls.append((a, k)), + ) + store._persisted_routing_generation = store._routing_generation + 1 + + store._save_entry(entry.session_key) + + assert calls == [] + store._db.close() + + def test_restart_rebind_after_skipped_idempotent_write( + self, tmp_path, monkeypatch + ): + """A skipped fast-path write never orphans the session on restart. + + The skip only fires when a newer FULL snapshot — which contains + this key — already persisted, so state.db can always rebind. + """ + store = _make_store(tmp_path, monkeypatch) + entry = store.get_or_create_session(_source()) + + calls = [] + monkeypatch.setattr( + store._db, + "save_gateway_routing_entry", + lambda *a, **k: calls.append(a), + ) + store._persisted_routing_generation = store._routing_generation + 1 + store._save_entry(entry.session_key) + assert calls == [] # the idempotent write was skipped + store._db.close() + + restarted = _make_store(tmp_path, monkeypatch) + rebound = restarted.get_or_create_session(_source()) + assert rebound.session_id == entry.session_id + restarted._db.close() + + def test_upsert_proceeds_at_current_generation(self, tmp_path, monkeypatch): + store = _make_store(tmp_path, monkeypatch) + entry = store.get_or_create_session(_source()) + + calls = [] + monkeypatch.setattr( + store._db, + "save_gateway_routing_entry", + lambda key, entry_json, **k: calls.append((key, entry_json, k)), + ) + + store._save_entry(entry.session_key) + + assert len(calls) == 1 + key, entry_json, kwargs = calls[0] + assert key == entry.session_key + assert json.loads(entry_json)["session_id"] == entry.session_id + assert kwargs["scope"] == store._routing_scope() + store._db.close() + + def test_missing_key_is_a_noop(self, tmp_path, monkeypatch): + store = _make_store(tmp_path, monkeypatch) + store._ensure_loaded() + + calls = [] + monkeypatch.setattr( + store._db, + "save_gateway_routing_entry", + lambda *a, **k: calls.append(a), + ) + + store._save_entry("agent:main:local:nope") + + assert calls == [] + store._db.close() + + +class _GatedSaveLock: + """``_save_lock`` wrapper that parks one thread at lock entry. + + The parked thread has already serialized its entry (and taken its + revision) under ``_lock``, so this deterministically reproduces a + delayed write: the writer sits between its serialize point and the + durable-write section while other writers run to completion. + """ + + def __init__(self, inner: threading.Lock) -> None: + self._inner = inner + self.gated_thread: threading.Thread | None = None + self.reached = threading.Event() + self.release = threading.Event() + + def __enter__(self): + if threading.current_thread() is self.gated_thread: + self.reached.set() + assert self.release.wait(timeout=5), "gated writer never released" + return self._inner.__enter__() + + def __exit__(self, *exc): + return self._inner.__exit__(*exc) + + +class TestDelayedWriteOrdering: + def test_reverse_completion_fast_saves_keep_newer_entry( + self, tmp_path, monkeypatch + ): + """Two same-key fast saves completing in reverse order. + + The older save serializes first but its UPSERT is delayed past + the newer save's. Its revision is below the newer one recorded + in ``_fast_persisted_entries``, so it must skip — the newer + entry_json stays in state.db. + """ + store = _make_store(tmp_path, monkeypatch) + entry = store.get_or_create_session(_source()) + + gate = _GatedSaveLock(store._save_lock) + store._save_lock = gate + older = threading.Thread( + target=store.update_session, + args=(entry.session_key,), + kwargs={"last_prompt_tokens": 1}, + ) + gate.gated_thread = older + older.start() + # The older save has serialized last_prompt_tokens=1 and parked + # before its UPSERT; the newer save now runs to completion. + assert gate.reached.wait(timeout=5) + store.update_session(entry.session_key, last_prompt_tokens=2) + + gate.release.set() + older.join(timeout=5) + assert not older.is_alive() + + assert _routing_row(store, entry.session_key)["last_prompt_tokens"] == 2 + store._db.close() + + def test_delayed_full_rewrite_folds_in_newer_fast_save( + self, tmp_path, monkeypatch + ): + """A full rewrite landing after a later-serialized fast save. + + The rewrite's snapshot predates the fast save, so replaying it + verbatim would regress the key. ``_persist_routing_data`` must + fold the newer fast record into the rewrite — in state.db and in + the sessions.json mirror. + """ + store = _make_store(tmp_path, monkeypatch) + entry = store.get_or_create_session(_source()) + + with store._lock: + data, generation = store._snapshot_routing_locked() + store.update_session(entry.session_key, last_prompt_tokens=7) + + # The delayed full rewrite lands last. + store._persist_routing_data(data, generation) + + assert _routing_row(store, entry.session_key)["last_prompt_tokens"] == 7 + sessions_json = tmp_path / "sessions" / "sessions.json" + mirror = json.loads(sessions_json.read_text(encoding="utf-8")) + assert mirror[entry.session_key]["last_prompt_tokens"] == 7 + store._db.close() + + def test_delayed_fast_save_skips_after_newer_full_rewrite( + self, tmp_path, monkeypatch + ): + """A fast save delayed past a full rewrite serialized after it. + + The rewrite's snapshot contains a newer copy of the key, so the + parked UPSERT must skip instead of regressing it. + """ + store = _make_store(tmp_path, monkeypatch) + entry = store.get_or_create_session(_source()) + + upserts = [] + real_saver = store._db.save_gateway_routing_entry + + def counting_saver(session_key, entry_json, **kwargs): + upserts.append(session_key) + real_saver(session_key, entry_json, **kwargs) + + monkeypatch.setattr( + store._db, "save_gateway_routing_entry", counting_saver + ) + gate = _GatedSaveLock(store._save_lock) + store._save_lock = gate + older = threading.Thread( + target=store.update_session, + args=(entry.session_key,), + kwargs={"last_prompt_tokens": 1}, + ) + gate.gated_thread = older + older.start() + assert gate.reached.wait(timeout=5) + + # A full rewrite serialized after the parked save persists first. + with store._lock: + store._entries[entry.session_key].last_prompt_tokens = 2 + store._save_entries() + + gate.release.set() + older.join(timeout=5) + assert not older.is_alive() + + assert upserts == [] # the delayed UPSERT was skipped + assert _routing_row(store, entry.session_key)["last_prompt_tokens"] == 2 + store._db.close() From 54eafee30b83e644abd950355e70f468ee01b8db Mon Sep 17 00:00:00 2001 From: kshitij <82637225+kshitijk4poor@users.noreply.github.com> Date: Sun, 2 Aug 2026 21:55:31 +0530 Subject: [PATCH 39/69] refactor(gateway): single allocator for the shared routing counter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-up on the #64169 salvage: _save_entry duplicated _snapshot_routing_locked's counter-bump line verbatim. The stale-write protection is a total order over ONE counter — extract _next_routing_generation_locked() so the two allocation sites can't drift apart silently. --- gateway/session.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/gateway/session.py b/gateway/session.py index c99b8a39cd96..92e2a5ae099d 100644 --- a/gateway/session.py +++ b/gateway/session.py @@ -1476,12 +1476,23 @@ def _save(self) -> None: data, generation = self._snapshot_routing_locked() self._persist_routing_data(data, generation) + def _next_routing_generation_locked(self) -> int: + """Bump and return the shared routing counter. Caller holds ``_lock``. + + BOTH full snapshots (_snapshot_routing_locked) and single-entry fast + saves (_save_entry) MUST allocate from this one counter — the stale- + write protection in _persist_routing_data/_save_entry is a total order + over serialization times and silently breaks if the two paths ever + number themselves independently. + """ + self._routing_generation = getattr(self, "_routing_generation", 0) + 1 + return self._routing_generation + def _snapshot_routing_locked(self) -> tuple[Dict[str, Any], int]: """Capture immutable routing data and a monotonic generation.""" - self._routing_generation = getattr(self, "_routing_generation", 0) + 1 return ( {key: entry.to_dict() for key, entry in self._entries.items()}, - self._routing_generation, + self._next_routing_generation_locked(), ) def _persist_routing_data(self, data: Dict[str, Any], generation: int) -> None: @@ -1623,8 +1634,7 @@ def _save_entry(self, session_key: str) -> None: if entry is None: return entry_json = json.dumps(entry.to_dict()) - self._routing_generation = getattr(self, "_routing_generation", 0) + 1 - revision = self._routing_generation + revision = self._next_routing_generation_locked() _db = getattr(self, "_db", None) saver = getattr(_db, "save_gateway_routing_entry", None) if _db else None if callable(saver): From 958ffd108525c7c469b463217bb67a740f0ff3e7 Mon Sep 17 00:00:00 2001 From: JabberELF Date: Fri, 22 May 2026 23:41:30 +0800 Subject: [PATCH 40/69] perf(memory): store holographic vectors as float32 --- plugins/memory/holographic/holographic.py | 69 +++++++- plugins/memory/holographic/retrieval.py | 16 +- plugins/memory/holographic/store.py | 2 +- .../test_holographic_vector_storage.py | 166 ++++++++++++++++++ 4 files changed, 235 insertions(+), 18 deletions(-) create mode 100644 tests/plugins/test_holographic_vector_storage.py diff --git a/plugins/memory/holographic/holographic.py b/plugins/memory/holographic/holographic.py index e1401fde108d..3aa5ddcdbe08 100644 --- a/plugins/memory/holographic/holographic.py +++ b/plugins/memory/holographic/holographic.py @@ -33,6 +33,7 @@ logger = logging.getLogger(__name__) _TWO_PI = 2.0 * math.pi +_FLOAT32_BLOB_PREFIX = b"HRR1" def _require_numpy() -> None: @@ -40,6 +41,12 @@ def _require_numpy() -> None: raise RuntimeError("numpy is required for holographic operations") +def _np(): + """Return the numpy module after the runtime availability guard.""" + _require_numpy() + return np # type: ignore[name-defined] + + def encode_atom(word: str, dim: int = 1024) -> "np.ndarray": """Deterministic phase vector via SHA-256 counter blocks. @@ -161,19 +168,63 @@ def encode_fact(content: str, entities: list[str], dim: int = 1024) -> "np.ndarr def phases_to_bytes(phases: "np.ndarray") -> bytes: - """Serialize phase vector to bytes. float64 tobytes — 8 KB at dim=1024.""" - _require_numpy() - return phases.tobytes() + """Serialize phase vectors as float32 blobs. + float32 halves SQLite BLOB storage versus the legacy float64 format + (4 KB + a 4-byte format prefix instead of 8 KB at dim=1024) while + preserving enough precision for phase-similarity retrieval. + ``bytes_to_phases`` keeps reading legacy float64 blobs for backward + compatibility. + """ + numpy = _np() + payload = numpy.asarray(phases, dtype=numpy.float32).tobytes() + return _FLOAT32_BLOB_PREFIX + payload -def bytes_to_phases(data: bytes) -> "np.ndarray": - """Deserialize bytes back to phase vector. Inverse of phases_to_bytes. - The .copy() call is required because frombuffer returns a read-only view - backed by the bytes object; callers expect a mutable array. +def bytes_to_phases(data: bytes, dim: int | None = None) -> "np.ndarray": + """Deserialize a phase vector from new float32 or legacy float64 storage. + + New float32 blobs carry a small prefix so callers can round-trip without + knowing ``dim``. Legacy float64 blobs are raw NumPy bytes and remain + readable for backward compatibility. The returned array is copied and + promoted to float64 so downstream HRR math keeps the existing numerical + behavior. """ - _require_numpy() - return np.frombuffer(data, dtype=np.float64).copy() + numpy = _np() + + if dim is not None: + float32_payload_bytes = dim * numpy.dtype(numpy.float32).itemsize + float32_blob_bytes = len(_FLOAT32_BLOB_PREFIX) + float32_payload_bytes + float64_bytes = dim * numpy.dtype(numpy.float64).itemsize + + if data.startswith(_FLOAT32_BLOB_PREFIX) and len(data) == float32_blob_bytes: + payload = data[len(_FLOAT32_BLOB_PREFIX):] + return numpy.frombuffer(payload, dtype=numpy.float32).astype(numpy.float64) + if len(data) == float64_bytes: + return numpy.frombuffer(data, dtype=numpy.float64).copy() + if data.startswith(_FLOAT32_BLOB_PREFIX): + payload_len = len(data) - len(_FLOAT32_BLOB_PREFIX) + raise ValueError( + f"HRR vector blob has {len(data)} bytes ({payload_len} payload bytes after " + f"the float32 prefix); expected {float32_blob_bytes} (prefixed float32) " + f"or {float64_bytes} (legacy float64) for dim={dim}" + ) + raise ValueError( + f"HRR legacy vector blob has {len(data)} bytes; expected " + f"{float64_bytes} (float64) for dim={dim}" + ) + + if data.startswith(_FLOAT32_BLOB_PREFIX): + payload = data[len(_FLOAT32_BLOB_PREFIX):] + if len(payload) % numpy.dtype(numpy.float32).itemsize != 0: + raise ValueError( + f"HRR float32 vector blob has invalid payload byte length: {len(payload)}" + ) + return numpy.frombuffer(payload, dtype=numpy.float32).astype(numpy.float64) + + if len(data) % numpy.dtype(numpy.float64).itemsize != 0: + raise ValueError(f"HRR legacy vector blob has invalid byte length: {len(data)}") + return numpy.frombuffer(data, dtype=numpy.float64).copy() def snr_estimate(dim: int, n_items: int) -> float: diff --git a/plugins/memory/holographic/retrieval.py b/plugins/memory/holographic/retrieval.py index 95b8f2ae95db..2bbcb71f473e 100644 --- a/plugins/memory/holographic/retrieval.py +++ b/plugins/memory/holographic/retrieval.py @@ -91,7 +91,7 @@ def search( # HRR similarity if self.hrr_weight > 0 and fact.get("hrr_vector"): - fact_vec = hrr.bytes_to_phases(fact["hrr_vector"]) + fact_vec = hrr.bytes_to_phases(fact["hrr_vector"], dim=self.hrr_dim) if query_vec is None: query_vec = hrr.encode_text(query, self.hrr_dim) hrr_sim = (hrr.similarity(query_vec, fact_vec) + 1.0) / 2.0 # shift to [0,1] @@ -154,7 +154,7 @@ def probe( (bank_name,), ).fetchone() if bank_row: - bank_vec = hrr.bytes_to_phases(bank_row["vector"]) + bank_vec = hrr.bytes_to_phases(bank_row["vector"], dim=self.hrr_dim) extracted = hrr.unbind(bank_vec, probe_key) # Use extracted signal to score individual facts return self._score_facts_by_vector( @@ -189,7 +189,7 @@ def probe( scored = [] for row in rows: fact = dict(row) - fact_vec = hrr.bytes_to_phases(fact.pop("hrr_vector")) + fact_vec = hrr.bytes_to_phases(fact.pop("hrr_vector"), dim=self.hrr_dim) # Unbind probe key from fact to see if entity is structurally present residual = hrr.unbind(fact_vec, probe_key) # Compare residual against content signal @@ -253,7 +253,7 @@ def related( scored = [] for row in rows: fact = dict(row) - fact_vec = hrr.bytes_to_phases(fact.pop("hrr_vector")) + fact_vec = hrr.bytes_to_phases(fact.pop("hrr_vector"), dim=self.hrr_dim) # Check structural similarity: unbind entity from fact residual = hrr.unbind(fact_vec, entity_vec) @@ -334,7 +334,7 @@ def reason( scored = [] for row in rows: fact = dict(row) - fact_vec = hrr.bytes_to_phases(fact.pop("hrr_vector")) + fact_vec = hrr.bytes_to_phases(fact.pop("hrr_vector"), dim=self.hrr_dim) entity_scores = [] for probe_key in entity_residuals: @@ -431,8 +431,8 @@ def contradict( continue # Not enough entity overlap to be contradictory # Content similarity via HRR vectors - v1 = hrr.bytes_to_phases(f1["hrr_vector"]) - v2 = hrr.bytes_to_phases(f2["hrr_vector"]) + v1 = hrr.bytes_to_phases(f1["hrr_vector"], dim=self.hrr_dim) + v2 = hrr.bytes_to_phases(f2["hrr_vector"], dim=self.hrr_dim) content_sim = hrr.similarity(v1, v2) # High entity overlap + low content similarity = potential contradiction @@ -484,7 +484,7 @@ def _score_facts_by_vector( scored = [] for row in rows: fact = dict(row) - fact_vec = hrr.bytes_to_phases(fact.pop("hrr_vector")) + fact_vec = hrr.bytes_to_phases(fact.pop("hrr_vector"), dim=self.hrr_dim) sim = hrr.similarity(target_vec, fact_vec) fact["score"] = (sim + 1.0) / 2.0 * fact["trust_score"] scored.append(fact) diff --git a/plugins/memory/holographic/store.py b/plugins/memory/holographic/store.py index bf58d7a22dbc..fcb193d4da22 100644 --- a/plugins/memory/holographic/store.py +++ b/plugins/memory/holographic/store.py @@ -561,7 +561,7 @@ def _rebuild_bank(self, category: str) -> None: self._conn.commit() return - vectors = [hrr.bytes_to_phases(row["hrr_vector"]) for row in rows] + vectors = [hrr.bytes_to_phases(row["hrr_vector"], dim=self.hrr_dim) for row in rows] bank_vector = hrr.bundle(*vectors) fact_count = len(vectors) diff --git a/tests/plugins/test_holographic_vector_storage.py b/tests/plugins/test_holographic_vector_storage.py new file mode 100644 index 000000000000..56415b7446bd --- /dev/null +++ b/tests/plugins/test_holographic_vector_storage.py @@ -0,0 +1,166 @@ +"""Storage-size regression tests for holographic HRR vectors.""" + +from __future__ import annotations + +import pytest + +np = pytest.importorskip("numpy") + +from plugins.memory.holographic import holographic as hrr +from plugins.memory.holographic.retrieval import FactRetriever +from plugins.memory.holographic.store import MemoryStore + + +pytestmark = pytest.mark.skipif( + not hrr._HAS_NUMPY, + reason="holographic vector storage requires numpy", +) + + +def _float32_blob_size(dim: int) -> int: + return len(hrr._FLOAT32_BLOB_PREFIX) + dim * np.dtype(np.float32).itemsize + + +def test_phases_to_bytes_stores_float32_and_round_trips_with_dim() -> None: + dim = 1024 + phases = hrr.encode_atom("storage-size-regression", dim=dim) + + blob = hrr.phases_to_bytes(phases) + + assert len(blob) == _float32_blob_size(dim) + restored = hrr.bytes_to_phases(blob, dim=dim) + assert restored.shape == (dim,) + np.testing.assert_allclose(restored, phases, rtol=0, atol=1e-6) + + +def test_phases_to_bytes_round_trips_without_dim() -> None: + dim = 1024 + phases = hrr.encode_atom("dimensionless-round-trip", dim=dim) + + restored = hrr.bytes_to_phases(hrr.phases_to_bytes(phases)) + + assert restored.shape == (dim,) + np.testing.assert_allclose(restored, phases, rtol=0, atol=1e-6) + + +def test_phases_to_bytes_round_trips_ambiguous_small_dims_without_dim() -> None: + dim = 2 + phases = hrr.encode_atom("ambiguous-small-dimension", dim=dim) + + restored = hrr.bytes_to_phases(hrr.phases_to_bytes(phases)) + + assert restored.shape == (dim,) + np.testing.assert_allclose(restored, phases, rtol=0, atol=1e-6) + + +def test_bytes_to_phases_rejects_malformed_float32_blobs() -> None: + phases = hrr.encode_atom("malformed-float32-blob", dim=2) + blob = hrr.phases_to_bytes(phases) + + with pytest.raises(ValueError, match="expected .* for dim=3"): + hrr.bytes_to_phases(blob, dim=3) + + with pytest.raises(ValueError, match="invalid payload byte length"): + hrr.bytes_to_phases(hrr._FLOAT32_BLOB_PREFIX + b"x") + + +def test_bytes_to_phases_reads_legacy_float64_blobs_with_and_without_dim() -> None: + dim = 1024 + phases = hrr.encode_atom("legacy-float64-regression", dim=dim) + legacy_blob = phases.astype(np.float64, copy=False).tobytes() + + assert len(legacy_blob) == dim * np.dtype(np.float64).itemsize + restored_with_dim = hrr.bytes_to_phases(legacy_blob, dim=dim) + restored_without_dim = hrr.bytes_to_phases(legacy_blob) + + assert restored_with_dim.shape == (dim,) + assert restored_without_dim.shape == (dim,) + np.testing.assert_allclose(restored_with_dim, phases, rtol=0, atol=0) + np.testing.assert_allclose(restored_without_dim, phases, rtol=0, atol=0) + + +def test_bytes_to_phases_prefers_dim_matched_legacy_float64_on_prefix_collision() -> None: + dim = 4 + legacy_blob = hrr._FLOAT32_BLOB_PREFIX + b"\0" * ( + dim * np.dtype(np.float64).itemsize - len(hrr._FLOAT32_BLOB_PREFIX) + ) + + restored = hrr.bytes_to_phases(legacy_blob, dim=dim) + + assert restored.shape == (dim,) + np.testing.assert_array_equal( + restored, + np.frombuffer(legacy_blob, dtype=np.float64).copy(), + ) + + +def test_memory_store_reads_legacy_float64_vectors(tmp_path) -> None: + dim = 64 + db_path = tmp_path / "legacy_memory_store.db" + + with MemoryStore(db_path=db_path, hrr_dim=dim) as store: + fact_id = store.add_fact( + 'Bob Stone keeps "legacy HRR vectors" searchable.', + category="compat", + tags="legacy storage", + ) + + fact_blob = store._conn.execute( + "SELECT hrr_vector FROM facts WHERE fact_id = ?", + (fact_id,), + ).fetchone()["hrr_vector"] + bank_blob = store._conn.execute( + "SELECT vector FROM memory_banks WHERE bank_name = ?", + ("cat:compat",), + ).fetchone()["vector"] + + legacy_fact_blob = hrr.bytes_to_phases(fact_blob, dim=dim).astype(np.float64).tobytes() + legacy_bank_blob = hrr.bytes_to_phases(bank_blob, dim=dim).astype(np.float64).tobytes() + store._conn.execute( + "UPDATE facts SET hrr_vector = ? WHERE fact_id = ?", + (legacy_fact_blob, fact_id), + ) + store._conn.execute( + "UPDATE memory_banks SET vector = ? WHERE bank_name = ?", + (legacy_bank_blob, "cat:compat"), + ) + store._conn.commit() + + assert len(legacy_fact_blob) == dim * np.dtype(np.float64).itemsize + assert len(legacy_bank_blob) == dim * np.dtype(np.float64).itemsize + + retriever = FactRetriever(store, hrr_dim=dim) + results = retriever.search("legacy HRR vectors", category="compat", limit=1) + + assert results + assert results[0]["fact_id"] == fact_id + + +def test_memory_store_persists_fact_and_bank_vectors_as_float32(tmp_path) -> None: + dim = 64 + db_path = tmp_path / "memory_store.db" + + with MemoryStore(db_path=db_path, hrr_dim=dim) as store: + fact_id = store.add_fact( + 'Alice Smith stores "compact HRR vectors" for Python tests.', + category="perf", + tags="hrr storage", + ) + + fact_blob = store._conn.execute( + "SELECT hrr_vector FROM facts WHERE fact_id = ?", + (fact_id,), + ).fetchone()["hrr_vector"] + bank_blob = store._conn.execute( + "SELECT vector FROM memory_banks WHERE bank_name = ?", + ("cat:perf",), + ).fetchone()["vector"] + + assert len(fact_blob) == _float32_blob_size(dim) + assert len(bank_blob) == _float32_blob_size(dim) + + retriever = FactRetriever(store, hrr_dim=dim) + results = retriever.search("compact HRR vectors", category="perf", limit=1) + + assert results + assert results[0]["fact_id"] == fact_id From 7a450ca5ce4682a0b20ecc31eca04af6cbd78206 Mon Sep 17 00:00:00 2001 From: JabberELF Date: Mon, 13 Jul 2026 19:53:51 +0800 Subject: [PATCH 41/69] fix(memory): resolve dim=1 float32/float64 blob ambiguity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When hrr_dim=1 the prefixed float32 blob (4+4=8 bytes) collides in size with a raw float64 blob (1×8=8 bytes), making the format discriminator in bytes_to_phases ambiguous — a legacy blob starting with HRR1 would be misread as a prefixed float32 vector. - phases_to_bytes now accepts an optional dim and falls back to writing raw float64 when the two blob sizes are equal. - bytes_to_phases prefers the legacy float64 interpretation when sizes collide and dim is provided, since phases_to_bytes never writes a prefixed float32 blob at dim=1. - Three regression tests cover dim=1 write, round-trip, and the legacy-prefix collision case. Addresses hermes-sweeper review on PR #30499. --- plugins/memory/holographic/holographic.py | 38 +++++++++++++++++- .../test_holographic_vector_storage.py | 39 +++++++++++++++++++ 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/plugins/memory/holographic/holographic.py b/plugins/memory/holographic/holographic.py index 3aa5ddcdbe08..31c94e10f51d 100644 --- a/plugins/memory/holographic/holographic.py +++ b/plugins/memory/holographic/holographic.py @@ -167,7 +167,7 @@ def encode_fact(content: str, entities: list[str], dim: int = 1024) -> "np.ndarr return bundle(*components) -def phases_to_bytes(phases: "np.ndarray") -> bytes: +def phases_to_bytes(phases: "np.ndarray", dim: int | None = None) -> bytes: """Serialize phase vectors as float32 blobs. float32 halves SQLite BLOB storage versus the legacy float64 format @@ -175,8 +175,20 @@ def phases_to_bytes(phases: "np.ndarray") -> bytes: preserving enough precision for phase-similarity retrieval. ``bytes_to_phases`` keeps reading legacy float64 blobs for backward compatibility. + + When ``dim`` is 1 the prefixed float32 blob (8 bytes) collides in size + with a raw float64 blob (8 bytes), making the format ambiguous. In + that case we fall back to writing raw float64 so that ``bytes_to_phases`` + can never misinterpret the blob. """ numpy = _np() + if dim is None: + dim = int(phases.shape[0]) + float32_blob_bytes = len(_FLOAT32_BLOB_PREFIX) + dim * numpy.dtype(numpy.float32).itemsize + float64_bytes = dim * numpy.dtype(numpy.float64).itemsize + if float32_blob_bytes == float64_bytes: + # dim=1: sizes collide, write legacy float64 to stay unambiguous + return numpy.asarray(phases, dtype=numpy.float64).tobytes() payload = numpy.asarray(phases, dtype=numpy.float32).tobytes() return _FLOAT32_BLOB_PREFIX + payload @@ -189,6 +201,13 @@ def bytes_to_phases(data: bytes, dim: int | None = None) -> "np.ndarray": readable for backward compatibility. The returned array is copied and promoted to float64 so downstream HRR math keeps the existing numerical behavior. + + When ``dim`` is 1 the prefixed float32 blob and the raw float64 blob are + both 8 bytes, so size alone cannot disambiguate. ``phases_to_bytes`` + avoids writing prefixed blobs in that case; here we guard the remaining + collision window (a legacy float64 blob that happens to start with the + ``HRR1`` prefix) by preferring the legacy interpretation when sizes + match and the caller supplied ``dim``. """ numpy = _np() @@ -197,6 +216,23 @@ def bytes_to_phases(data: bytes, dim: int | None = None) -> "np.ndarray": float32_blob_bytes = len(_FLOAT32_BLOB_PREFIX) + float32_payload_bytes float64_bytes = dim * numpy.dtype(numpy.float64).itemsize + # When sizes collide (dim=1), prefer legacy float64 for a blob that + # starts with the prefix, because phases_to_bytes never writes a + # prefixed float32 blob at dim=1 — any such blob must be legacy. + if float32_blob_bytes == float64_bytes: + if len(data) == float64_bytes: + return numpy.frombuffer(data, dtype=numpy.float64).copy() + if data.startswith(_FLOAT32_BLOB_PREFIX): + payload_len = len(data) - len(_FLOAT32_BLOB_PREFIX) + raise ValueError( + f"HRR vector blob has {len(data)} bytes ({payload_len} payload bytes after " + f"the float32 prefix); expected {float64_bytes} (legacy float64) for dim={dim}" + ) + raise ValueError( + f"HRR legacy vector blob has {len(data)} bytes; expected " + f"{float64_bytes} (float64) for dim={dim}" + ) + if data.startswith(_FLOAT32_BLOB_PREFIX) and len(data) == float32_blob_bytes: payload = data[len(_FLOAT32_BLOB_PREFIX):] return numpy.frombuffer(payload, dtype=numpy.float32).astype(numpy.float64) diff --git a/tests/plugins/test_holographic_vector_storage.py b/tests/plugins/test_holographic_vector_storage.py index 56415b7446bd..70e984ab25f8 100644 --- a/tests/plugins/test_holographic_vector_storage.py +++ b/tests/plugins/test_holographic_vector_storage.py @@ -94,6 +94,45 @@ def test_bytes_to_phases_prefers_dim_matched_legacy_float64_on_prefix_collision( ) +def test_dim1_phases_to_bytes_writes_legacy_float64() -> None: + """At dim=1 the float32 prefixed blob (8 B) collides with raw float64 + (8 B), so phases_to_bytes must fall back to raw float64.""" + dim = 1 + phases = hrr.encode_atom("dim-one-ambiguity", dim=dim) + + blob = hrr.phases_to_bytes(phases, dim=dim) + + assert len(blob) == dim * np.dtype(np.float64).itemsize # 8 bytes, no prefix + assert not blob.startswith(hrr._FLOAT32_BLOB_PREFIX) + + +def test_dim1_round_trip_with_dim() -> None: + """Round-trip at dim=1 must work via the legacy float64 path.""" + dim = 1 + phases = hrr.encode_atom("dim-one-round-trip", dim=dim) + + restored = hrr.bytes_to_phases(hrr.phases_to_bytes(phases, dim=dim), dim=dim) + + assert restored.shape == (dim,) + np.testing.assert_allclose(restored, phases, rtol=0, atol=0) + + +def test_dim1_legacy_blob_starting_with_prefix_decodes_as_float64() -> None: + """A legacy float64 blob at dim=1 that happens to start with HRR1 must + decode as float64, not be misread as a prefixed float32 blob.""" + dim = 1 + phases = hrr.encode_atom("prefix-collision-dim-one", dim=dim) + legacy_blob = phases.astype(np.float64).tobytes() + # Force the blob to start with HRR1 prefix bytes + collision_blob = hrr._FLOAT32_BLOB_PREFIX + legacy_blob[len(hrr._FLOAT32_BLOB_PREFIX):] + assert len(collision_blob) == dim * np.dtype(np.float64).itemsize + + restored = hrr.bytes_to_phases(collision_blob, dim=dim) + + assert restored.shape == (dim,) + np.testing.assert_allclose(restored, np.frombuffer(collision_blob, dtype=np.float64).copy(), rtol=0, atol=0) + + def test_memory_store_reads_legacy_float64_vectors(tmp_path) -> None: dim = 64 db_path = tmp_path / "legacy_memory_store.db" From 94b10eccf5d444a272bbd52d8e662d8f8aec843c Mon Sep 17 00:00:00 2001 From: And Date: Sat, 11 Jul 2026 21:04:05 +0300 Subject: [PATCH 42/69] feat(memory/hindsight): order background prefetch after pending retains Async retain already keeps the memory WRITE off the reply path (writes drain on the single writer thread while the user gets their response immediately). This closes the remaining retain/prefetch race: the next turn's warm prefetch runs on its own thread and could recall BEFORE the just-enqueued retain write lands, silently dropping the latest turn from recall. - The background prefetch now waits (bounded) for pending retains to drain before recalling, so warmed context includes the just-completed turn. - The wait runs only on the background prefetch thread, never the reply path, so it adds zero latency to the user's response and loses no writes. - Bounded by prefetch_retain_drain_timeout (default 10s) and polls unfinished_tasks so a wedged write can't hang the prefetch. - New config keys: prefetch_waits_for_retain (default true), prefetch_retain_drain_timeout (default 10.0). --- plugins/memory/hindsight/__init__.py | 47 +++++++++++++++++++ .../plugins/memory/test_hindsight_provider.py | 41 ++++++++++++++++ 2 files changed, 88 insertions(+) diff --git a/plugins/memory/hindsight/__init__.py b/plugins/memory/hindsight/__init__.py index 03666f2c4180..b64992a4253b 100644 --- a/plugins/memory/hindsight/__init__.py +++ b/plugins/memory/hindsight/__init__.py @@ -39,6 +39,7 @@ import queue import sys import threading +import time from datetime import datetime, timezone from typing import Any, Dict, List @@ -742,6 +743,14 @@ def __init__(self): self._auto_retain = True self._retain_every_n_turns = 1 self._retain_async = True + # Async retain never blocks the reply (writes drain on the single + # writer thread). But the next turn's warm prefetch runs on its own + # thread and could read BEFORE the just-enqueued retain write lands, + # dropping the latest turn from recall. When True, the background + # prefetch waits (bounded) for pending retains to drain first, closing + # that race without putting any write on the reply path. + self._prefetch_waits_for_retain = True + self._prefetch_retain_drain_timeout = 10.0 self._retain_context = "conversation between Hermes Agent and the User" self._turn_counter = 0 self._session_turns: list[str] = [] # accumulates ALL turns for the session @@ -1058,6 +1067,8 @@ def get_config_schema(self): {"key": "auto_retain", "description": "Automatically retain conversation turns", "default": True}, {"key": "retain_every_n_turns", "description": "Retain every N turns (1 = every turn)", "default": 1}, {"key": "retain_async","description": "Process retain asynchronously on the Hindsight server", "default": True}, + {"key": "prefetch_waits_for_retain", "description": "Have the background next-turn prefetch wait for pending retain writes to drain first, so recall includes the just-completed turn (runs off the reply path, adds no response latency)", "default": True}, + {"key": "prefetch_retain_drain_timeout", "description": "Max seconds the background prefetch waits for pending retains to drain before recalling anyway", "default": 10.0}, {"key": "retain_context", "description": "Context label for retained memories", "default": "conversation between Hermes Agent and the User"}, {"key": "recall_max_tokens", "description": "Maximum tokens for recall results", "default": 4096}, {"key": "recall_max_input_chars", "description": "Maximum input query length for auto-recall", "default": 800}, @@ -1162,6 +1173,32 @@ def _ensure_writer(self) -> None: self._sync_thread = thread thread.start() + def _wait_for_retains_drained(self, timeout: float) -> bool: + """Block up to *timeout* seconds for queued retains to finish. + + Used by the background prefetch so the next turn's recall observes + the just-completed turn's write instead of racing ahead of it. Runs + only on the background prefetch thread — never on the reply path. + + Returns True if the queue drained within the budget, False on timeout. + Polls ``unfinished_tasks`` rather than ``queue.join()`` so the wait is + bounded even if a write is wedged (the writer is best-effort). + """ + if timeout <= 0: + return self._retain_queue.unfinished_tasks == 0 + deadline = time.monotonic() + timeout + while self._retain_queue.unfinished_tasks > 0: + if self._shutting_down.is_set(): + return False + if time.monotonic() >= deadline: + logger.debug( + "Prefetch: retain drain timed out after %.1fs (%d pending)", + timeout, self._retain_queue.unfinished_tasks, + ) + return False + time.sleep(0.05) + return True + def _writer_loop(self) -> None: """Drain the retain queue serially. Exits on sentinel. @@ -1404,6 +1441,10 @@ def initialize(self, session_id: str, **kwargs) -> None: self._recall_prompt_preamble = self._config.get("recall_prompt_preamble", "") self._recall_max_input_chars = int(self._config.get("recall_max_input_chars", 800)) self._retain_async = self._config.get("retain_async", True) + self._prefetch_waits_for_retain = self._config.get("prefetch_waits_for_retain", True) + self._prefetch_retain_drain_timeout = float( + self._config.get("prefetch_retain_drain_timeout", 10.0) + ) _client_version = "unknown" try: @@ -1548,6 +1589,12 @@ def queue_prefetch(self, query: str, *, session_id: str = "") -> None: query = query[:self._recall_max_input_chars] def _run(): + # Ensure the just-completed turn's retain has landed before we + # recall, so the warmed context for the next turn includes it. + # This runs on the background prefetch thread, never the reply + # path, so it adds no latency to the user's response. + if self._prefetch_waits_for_retain: + self._wait_for_retains_drained(self._prefetch_retain_drain_timeout) try: if self._prefetch_method == "reflect": logger.debug("Prefetch: calling reflect (bank=%s, query_len=%d)", self._bank_id, len(query)) diff --git a/tests/plugins/memory/test_hindsight_provider.py b/tests/plugins/memory/test_hindsight_provider.py index 0c72bd6203df..092061830ea2 100644 --- a/tests/plugins/memory/test_hindsight_provider.py +++ b/tests/plugins/memory/test_hindsight_provider.py @@ -10,6 +10,7 @@ import re import stat import sys +import time from pathlib import Path from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock @@ -501,6 +502,46 @@ def test_queue_prefetch_skipped_in_tools_mode(self, provider_with_config): # Should not start a thread assert p._prefetch_thread is None + def test_prefetch_waits_for_pending_retain_before_recall(self, provider): + """The background prefetch must wait for queued retains to drain so the + next turn's recall observes the just-completed turn (no retain race).""" + import threading + + order = [] + release = threading.Event() + + async def _slow_retain(*args, **kwargs): + release.wait(timeout=5.0) + order.append("retain") + + async def _recall(**kwargs): + order.append("recall") + return SimpleNamespace(results=[SimpleNamespace(text="m")]) + + provider._client.aretain_batch = AsyncMock(side_effect=_slow_retain) + provider._client.arecall = AsyncMock(side_effect=_recall) + + # Enqueue a slow retain, then immediately queue the next-turn prefetch. + provider.sync_turn("hello", "world") + provider.queue_prefetch("next turn query") + + # Let the prefetch thread start and reach the drain barrier. + time.sleep(0.2) + assert order == [], "recall ran before the pending retain drained" + + # Release the retain; the prefetch should now proceed AFTER it. + release.set() + if provider._prefetch_thread: + provider._prefetch_thread.join(timeout=5.0) + provider._retain_queue.join() + assert order and order[0] == "retain" + assert "recall" in order + + def test_prefetch_wait_for_retain_can_be_disabled(self, provider_with_config): + p = provider_with_config(prefetch_waits_for_retain=False) + p._client = _make_mock_client() + assert p._prefetch_waits_for_retain is False + # --------------------------------------------------------------------------- # sync_turn tests From 1be353bf9c1eae871f55597810e2c65bbd6472d1 Mon Sep 17 00:00:00 2001 From: And Date: Mon, 13 Jul 2026 02:30:05 +0300 Subject: [PATCH 43/69] fix(memory/hindsight): gate prefetch on server-side retain completion, not just queue drain Address PR #62871 review: with the default retain_async=True, aretain_batch returns when the write is accepted, not when it's durable/recall-visible, so draining the local writer queue (task_done) is not a read-after-write signal. The next-turn prefetch could still recall before the just-completed turn was observable on the server. - Track the async operation_id/operation_ids returned by aretain_batch - _wait_for_retains_drained now applies two ordered, budget-bounded barriers: (1) local writer queue drains, then (2) tracked server-side async ops report completion via operations.get_operation_status (an explicit read-after-write condition). NotFound (completed+evicted) counts as done; transient errors keep waiting until the deadline - Completed ops are removed from the pending set so later prefetches don't re-poll them; the whole wait stays off the reply path - Add TestPrefetchServerRetainVisibility: op-id tracking (single/multiple), no-op tracking when retain_async=False, prefetch waiting for server completion before recall, timeout fallback on a wedged op, and NotFound / transient-error status handling --- plugins/memory/hindsight/__init__.py | 174 +++++++++++++++--- .../plugins/memory/test_hindsight_provider.py | 131 +++++++++++++ 2 files changed, 281 insertions(+), 24 deletions(-) diff --git a/plugins/memory/hindsight/__init__.py b/plugins/memory/hindsight/__init__.py index b64992a4253b..c12238518c06 100644 --- a/plugins/memory/hindsight/__init__.py +++ b/plugins/memory/hindsight/__init__.py @@ -727,6 +727,16 @@ def __init__(self): self._writer_thread: threading.Thread | None = None self._shutting_down = threading.Event() self._atexit_registered = False + # Server-side async retain operations still in flight. With + # retain_async=True, aretain_batch returns as soon as the write is + # *accepted*, not when it's durable/recall-visible, so the returned + # operation_id(s) stay "pending" until the server finishes. The + # background prefetch gates on these via get_operation_status so recall + # observes the just-completed turn (draining the local queue alone is + # not a read-after-write signal for async retains). + self._pending_retain_ops: set[str] = set() + self._pending_retain_ops_lock = threading.Lock() + self._retain_ops_bank_id = "" # Legacy alias — older tests/callers reference _sync_thread directly. # Points at _writer_thread once the writer is running. self._sync_thread = None @@ -745,10 +755,13 @@ def __init__(self): self._retain_async = True # Async retain never blocks the reply (writes drain on the single # writer thread). But the next turn's warm prefetch runs on its own - # thread and could read BEFORE the just-enqueued retain write lands, - # dropping the latest turn from recall. When True, the background - # prefetch waits (bounded) for pending retains to drain first, closing - # that race without putting any write on the reply path. + # thread and could read BEFORE the just-completed retain is + # recall-visible on the server, dropping the latest turn from recall. + # When True, the background prefetch first waits (bounded) for the + # local writer queue to drain AND for the server-side async retain + # operation(s) to report completion, an explicit read-after-write + # signal — closing that race without putting any write on the reply + # path. self._prefetch_waits_for_retain = True self._prefetch_retain_drain_timeout = 10.0 self._retain_context = "conversation between Hermes Agent and the User" @@ -1067,8 +1080,8 @@ def get_config_schema(self): {"key": "auto_retain", "description": "Automatically retain conversation turns", "default": True}, {"key": "retain_every_n_turns", "description": "Retain every N turns (1 = every turn)", "default": 1}, {"key": "retain_async","description": "Process retain asynchronously on the Hindsight server", "default": True}, - {"key": "prefetch_waits_for_retain", "description": "Have the background next-turn prefetch wait for pending retain writes to drain first, so recall includes the just-completed turn (runs off the reply path, adds no response latency)", "default": True}, - {"key": "prefetch_retain_drain_timeout", "description": "Max seconds the background prefetch waits for pending retains to drain before recalling anyway", "default": 10.0}, + {"key": "prefetch_waits_for_retain", "description": "Have the background next-turn prefetch wait for the just-completed retain to become recall-visible on the server (local queue drain + async operation completion) before recalling, so recall includes the just-completed turn (runs off the reply path, adds no response latency)", "default": True}, + {"key": "prefetch_retain_drain_timeout", "description": "Max seconds the background prefetch waits for the retain to become recall-visible (queue drain + server-side completion) before recalling anyway", "default": 10.0}, {"key": "retain_context", "description": "Context label for retained memories", "default": "conversation between Hermes Agent and the User"}, {"key": "recall_max_tokens", "description": "Maximum tokens for recall results", "default": 4096}, {"key": "recall_max_input_chars", "description": "Maximum input query length for auto-recall", "default": 800}, @@ -1173,31 +1186,136 @@ def _ensure_writer(self) -> None: self._sync_thread = thread thread.start() - def _wait_for_retains_drained(self, timeout: float) -> bool: - """Block up to *timeout* seconds for queued retains to finish. + def _track_retain_ops(self, retain_response, bank_id: str) -> None: + """Record server-side async operation id(s) from an aretain_batch reply. + + Async retains return ``operation_id`` / ``operation_ids`` that stay + ``pending`` on the server until the write is durable and recall-visible. + The bank_id is captured alongside so completion can be polled with the + same bank the write targeted. + """ + ids: list[str] = [] + single = getattr(retain_response, "operation_id", None) + if single: + ids.append(str(single)) + multiple = getattr(retain_response, "operation_ids", None) + if multiple: + ids.extend(str(op) for op in multiple if op) + if not ids: + # Server didn't hand back an op id (older API, or it completed + # synchronously). Nothing to poll — local queue drain is the only + # available signal in that case. + return + self._retain_ops_bank_id = bank_id + with self._pending_retain_ops_lock: + self._pending_retain_ops.update(ids) - Used by the background prefetch so the next turn's recall observes - the just-completed turn's write instead of racing ahead of it. Runs - only on the background prefetch thread — never on the reply path. + def _is_retain_op_complete(self, bank_id: str, op_id: str) -> bool: + """Return True when a server-side async retain op is done (or gone). - Returns True if the queue drained within the budget, False on timeout. - Polls ``unfinished_tasks`` rather than ``queue.join()`` so the wait is - bounded even if a write is wedged (the writer is best-effort). + ``get_operation_status`` returns ``completed``/``failed`` for a known + op; completed ops are evicted server-side, so a NotFound (404) also + means "no longer pending" and is treated as done. Transient errors + return False so the caller keeps waiting until its deadline. """ - if timeout <= 0: - return self._retain_queue.unfinished_tasks == 0 - deadline = time.monotonic() + timeout + from hindsight_client_api.exceptions import NotFoundException + + try: + resp = self._run_hindsight_operation( + lambda client: client.operations.get_operation_status( + bank_id=bank_id, operation_id=op_id + ) + ) + except NotFoundException: + return True + except Exception as exc: + logger.debug("Prefetch: operation status check failed for %s: %s", op_id, exc) + return False + status = str(getattr(resp, "status", "") or "").lower() + return status in {"completed", "failed"} + + def _wait_for_retains_drained(self, timeout: float) -> bool: + """Block up to *timeout* seconds for the just-completed turn's retain to + become recall-visible on the server. + + Used by the background prefetch so the next turn's recall observes the + just-completed turn's write instead of racing ahead of it. Runs only on + the background prefetch thread — never on the reply path. + + Two ordered barriers, both bounded by the shared *timeout* budget: + + 1. Local writer queue drains (the retain call has been *dispatched* to + the server). Polls ``unfinished_tasks`` rather than ``queue.join()`` + so a wedged write can't hang the prefetch. + 2. Server-side async operations complete. With ``retain_async=True`` the + dispatched call returns on *acceptance*, not durability, so draining + the local queue alone is NOT a read-after-write signal. We poll + ``get_operation_status`` for the tracked op id(s) until the server + reports completion (an explicit read-after-write condition). + + Returns True if both barriers cleared within the budget, False on + timeout/shutdown. + """ + deadline = None if timeout <= 0 else time.monotonic() + timeout + + def _expired() -> bool: + return deadline is not None and time.monotonic() >= deadline + + # Barrier 1: local queue drain (retain dispatched to the server). while self._retain_queue.unfinished_tasks > 0: if self._shutting_down.is_set(): return False - if time.monotonic() >= deadline: + if _expired(): logger.debug( "Prefetch: retain drain timed out after %.1fs (%d pending)", timeout, self._retain_queue.unfinished_tasks, ) return False time.sleep(0.05) - return True + + # Barrier 2: server-side async retain completion (read-after-write). + return self._wait_for_server_retain_ops(deadline, timeout) + + def _wait_for_server_retain_ops(self, deadline: float | None, timeout: float) -> bool: + """Poll tracked async retain ops until complete or the deadline passes. + + *deadline* is a ``time.monotonic()`` value (None = no bound). Completed + ops are removed from the pending set as they finish so a later prefetch + doesn't re-poll them. + """ + while True: + with self._pending_retain_ops_lock: + bank_id = getattr(self, "_retain_ops_bank_id", "") or self._bank_id + pending = list(self._pending_retain_ops) + if not pending: + return True + if self._shutting_down.is_set(): + return False + + done: set[str] = set() + for op_id in pending: + if self._shutting_down.is_set(): + return False + if deadline is not None and time.monotonic() >= deadline: + logger.debug( + "Prefetch: server retain visibility timed out after %.1fs " + "(%d op(s) still pending)", + timeout, len(pending) - len(done), + ) + with self._pending_retain_ops_lock: + self._pending_retain_ops.difference_update(done) + return False + if self._is_retain_op_complete(bank_id, op_id): + done.add(op_id) + + with self._pending_retain_ops_lock: + self._pending_retain_ops.difference_update(done) + still_pending = bool(self._pending_retain_ops) + if not still_pending: + return True + if deadline is not None and time.monotonic() >= deadline: + return False + time.sleep(0.05) def _writer_loop(self) -> None: """Drain the retain queue serially. Exits on sentinel. @@ -1589,10 +1707,13 @@ def queue_prefetch(self, query: str, *, session_id: str = "") -> None: query = query[:self._recall_max_input_chars] def _run(): - # Ensure the just-completed turn's retain has landed before we - # recall, so the warmed context for the next turn includes it. - # This runs on the background prefetch thread, never the reply - # path, so it adds no latency to the user's response. + # Ensure the just-completed turn's retain is recall-visible on the + # server before we recall, so the warmed context for the next turn + # includes it. This waits for the local writer queue to drain AND + # for the server-side async retain op(s) to complete (an explicit + # read-after-write signal), because async retain returns on + # acceptance rather than durability. Runs on the background prefetch + # thread, never the reply path, so it adds no response latency. if self._prefetch_waits_for_retain: self._wait_for_retains_drained(self._prefetch_retain_drain_timeout) try: @@ -1776,7 +1897,7 @@ def _do_retain() -> None: item["update_mode"] = update_mode logger.debug("Hindsight retain: bank=%s, doc=%s, mode=%s, async=%s, content_len=%d, num_turns=%d", bank_id, document_id, update_mode, retain_async_flag, len(content), num_turns) - self._run_hindsight_operation( + resp = self._run_hindsight_operation( lambda client: client.aretain_batch( bank_id=bank_id, items=[item], @@ -1784,6 +1905,11 @@ def _do_retain() -> None: retain_async=retain_async_flag, ) ) + # For async retains the write is only *accepted* here; track the + # returned operation id(s) so the next-turn prefetch can wait for + # true server-side completion (read-after-write) before recalling. + if retain_async_flag: + self._track_retain_ops(resp, bank_id) logger.debug("Hindsight retain succeeded") self._ensure_writer() diff --git a/tests/plugins/memory/test_hindsight_provider.py b/tests/plugins/memory/test_hindsight_provider.py index 092061830ea2..fef2bf3e7f5a 100644 --- a/tests/plugins/memory/test_hindsight_provider.py +++ b/tests/plugins/memory/test_hindsight_provider.py @@ -543,6 +543,137 @@ def test_prefetch_wait_for_retain_can_be_disabled(self, provider_with_config): assert p._prefetch_waits_for_retain is False +class TestPrefetchServerRetainVisibility: + """PR #62871 review follow-up: draining the local writer queue is not a + read-after-write signal for async retains. With ``retain_async=True`` the + server accepts the write and returns an ``operation_id`` that stays + ``pending`` until the write is durable/recall-visible. The background + prefetch must gate on server-side operation completion, not just the local + queue, before recalling. + """ + + def _client_with_ops(self, statuses): + """Mock client whose aretain_batch returns an async operation_id and + whose operations.get_operation_status yields *statuses* in order + (last value repeats).""" + client = _make_mock_client() + client.aretain_batch = AsyncMock( + return_value=SimpleNamespace(operation_id="op-1", operation_ids=None) + ) + seq = list(statuses) + + async def _status(**kwargs): + value = seq.pop(0) if len(seq) > 1 else seq[0] + return SimpleNamespace(status=value) + + client.operations = MagicMock() + client.operations.get_operation_status = AsyncMock(side_effect=_status) + return client + + def test_tracks_async_operation_id_from_retain(self, provider): + provider._client.aretain_batch = AsyncMock( + return_value=SimpleNamespace(operation_id="op-async-1", operation_ids=None) + ) + provider.sync_turn("hello", "world") + provider._retain_queue.join() + assert "op-async-1" in provider._pending_retain_ops + + def test_tracks_multiple_operation_ids(self, provider): + provider._client.aretain_batch = AsyncMock( + return_value=SimpleNamespace( + operation_id=None, operation_ids=["op-a", "op-b"] + ) + ) + provider.sync_turn("hello", "world") + provider._retain_queue.join() + assert {"op-a", "op-b"} <= provider._pending_retain_ops + + def test_sync_retain_tracks_no_ops(self, provider_with_config): + p = provider_with_config(retain_async=False) + p._client = _make_mock_client() + p._client.aretain_batch = AsyncMock( + return_value=SimpleNamespace(operation_id="op-x", operation_ids=None) + ) + p.sync_turn("hello", "world") + p._retain_queue.join() + # retain_async=False → no server-side op to wait on. + assert p._pending_retain_ops == set() + + def test_prefetch_waits_for_server_completion_before_recall(self, provider): + """Recall must not run until the tracked async op reports completed.""" + order = [] + + async def _recall(**kwargs): + order.append("recall") + return SimpleNamespace(results=[SimpleNamespace(text="m")]) + + provider._client = self._client_with_ops(["pending", "pending", "completed"]) + provider._client.arecall = AsyncMock(side_effect=_recall) + + provider.sync_turn("hello", "world") + provider._retain_queue.join() + assert "op-1" in provider._pending_retain_ops + + provider.queue_prefetch("next turn query") + if provider._prefetch_thread: + provider._prefetch_thread.join(timeout=5.0) + + # Recall ran, the op was polled to completion, and the pending set + # was cleared (so a later prefetch won't re-poll it). + assert order == ["recall"] + assert provider._client.operations.get_operation_status.await_count >= 3 + assert provider._pending_retain_ops == set() + + def test_prefetch_proceeds_after_server_wait_timeout(self, provider_with_config): + """A wedged/never-completing async op must not hang prefetch forever; + it recalls anyway once the drain budget is exhausted.""" + p = provider_with_config(prefetch_retain_drain_timeout=0.3) + order = [] + + async def _recall(**kwargs): + order.append("recall") + return SimpleNamespace(results=[SimpleNamespace(text="m")]) + + p._client = self._client_with_ops(["pending"]) # never completes + p._client.arecall = AsyncMock(side_effect=_recall) + + p.sync_turn("hello", "world") + p._retain_queue.join() + + start = time.monotonic() + p.queue_prefetch("next turn query") + if p._prefetch_thread: + p._prefetch_thread.join(timeout=5.0) + elapsed = time.monotonic() - start + + assert order == ["recall"], "prefetch should recall after the timeout" + assert elapsed < 3.0, "prefetch must not block well past the drain budget" + + def test_operation_notfound_treated_as_complete(self, provider): + """A NotFound (completed+evicted) op is treated as done, not pending.""" + from hindsight_client_api.exceptions import NotFoundException + + client = _make_mock_client() + client.operations = MagicMock() + client.operations.get_operation_status = AsyncMock( + side_effect=NotFoundException(status=404, reason="gone") + ) + provider._client = client + + assert provider._is_retain_op_complete("bank", "op-gone") is True + + def test_transient_status_error_keeps_waiting(self, provider): + """A transient status-check error means 'unknown', so keep waiting.""" + client = _make_mock_client() + client.operations = MagicMock() + client.operations.get_operation_status = AsyncMock( + side_effect=RuntimeError("temporary blip") + ) + provider._client = client + + assert provider._is_retain_op_complete("bank", "op-1") is False + + # --------------------------------------------------------------------------- # sync_turn tests # --------------------------------------------------------------------------- From 9bbd956b739d9997751d0b33bcac4274f35d00a6 Mon Sep 17 00:00:00 2001 From: kshitij <82637225+kshitijk4poor@users.noreply.github.com> Date: Sun, 2 Aug 2026 21:54:40 +0530 Subject: [PATCH 44/69] fix(memory/hindsight): evict timed-out retain ops + coarser status polls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-up on the #62871 salvage (simplify pass, HIGH): 1. Ops unresolved at the wait deadline were RETAINED in the pending set. A permanently failing status endpoint (auth error, endless 500s, or a server that loses ops without 404) would grow the set forever and make EVERY later prefetch burn the full 10s budget re-polling it — and prefetch()'s bounded 3s join sits on the reply path, so that money-quote 'adds no response latency' claim breaks. Timed-out ops are now dropped (identical degradation to prefetch_waits_for_retain=False: possibly stale recall) with a WARNING so persistent server trouble is visible. Guard test mutation-checked (fails with eviction disabled). 2. Status polls now spaced 0.5s (was 0.05s shared with the local drain poll): a wedged op cost up to ~200 get_operation_status round trips per prefetch; now ~20 max over the default 10s budget. --- plugins/memory/hindsight/__init__.py | 55 ++++++++++++++++--- .../plugins/memory/test_hindsight_provider.py | 32 +++++++++++ 2 files changed, 78 insertions(+), 9 deletions(-) diff --git a/plugins/memory/hindsight/__init__.py b/plugins/memory/hindsight/__init__.py index c12238518c06..8c93df87f6c3 100644 --- a/plugins/memory/hindsight/__init__.py +++ b/plugins/memory/hindsight/__init__.py @@ -737,6 +737,11 @@ def __init__(self): self._pending_retain_ops: set[str] = set() self._pending_retain_ops_lock = threading.Lock() self._retain_ops_bank_id = "" + # Seconds between get_operation_status polls while waiting for server- + # side retain completion. Each poll is a server round trip, so this is + # deliberately coarser than the 0.05s local queue-drain poll: ~20 calls + # max over the default 10s budget instead of ~200. + self._RETAIN_OP_POLL_INTERVAL_S = 0.5 # Legacy alias — older tests/callers reference _sync_thread directly. # Points at _writer_thread once the writer is running. self._sync_thread = None @@ -1282,6 +1287,21 @@ def _wait_for_server_retain_ops(self, deadline: float | None, timeout: float) -> *deadline* is a ``time.monotonic()`` value (None = no bound). Completed ops are removed from the pending set as they finish so a later prefetch doesn't re-poll them. + + Ops still pending when the deadline expires are DROPPED, not retained: + keeping them would make a permanently failing status endpoint (auth + error, endless 500s, server that loses ops without a 404) grow the + pending set forever and burn the full timeout on EVERY subsequent + prefetch — turning "bounded wait per prefetch" into unbounded + session-wide degradation (and, via prefetch()'s bounded join on the + reply path, a per-turn reply-latency penalty). Dropping trades a + possibly-stale recall NOW (identical to prefetch_waits_for_retain=False + behavior) for guaranteed liveness; the drop is logged at WARNING once + per prefetch so persistent server trouble is visible. + + Status polls are spaced by _RETAIN_OP_POLL_INTERVAL_S (0.5s) — server + round trips per op are bounded (~20 over a 10s budget), unlike the + cheap 0.05s local queue-drain poll in _wait_for_retains_drained. """ while True: with self._pending_retain_ops_lock: @@ -1293,29 +1313,46 @@ def _wait_for_server_retain_ops(self, deadline: float | None, timeout: float) -> return False done: set[str] = set() + expired = False for op_id in pending: if self._shutting_down.is_set(): return False if deadline is not None and time.monotonic() >= deadline: - logger.debug( - "Prefetch: server retain visibility timed out after %.1fs " - "(%d op(s) still pending)", - timeout, len(pending) - len(done), - ) - with self._pending_retain_ops_lock: - self._pending_retain_ops.difference_update(done) - return False + expired = True + break if self._is_retain_op_complete(bank_id, op_id): done.add(op_id) + if expired: + with self._pending_retain_ops_lock: + self._pending_retain_ops.difference_update(done) + dropped = len(self._pending_retain_ops) + self._pending_retain_ops.clear() + logger.warning( + "Prefetch: server retain visibility timed out after %.1fs; " + "dropping %d unresolved op(s) so later prefetches stay " + "bounded (recall may miss the just-completed turn)", + timeout, dropped, + ) + return False + with self._pending_retain_ops_lock: self._pending_retain_ops.difference_update(done) still_pending = bool(self._pending_retain_ops) if not still_pending: return True if deadline is not None and time.monotonic() >= deadline: + with self._pending_retain_ops_lock: + dropped = len(self._pending_retain_ops) + self._pending_retain_ops.clear() + logger.warning( + "Prefetch: server retain visibility timed out after %.1fs; " + "dropping %d unresolved op(s) so later prefetches stay " + "bounded (recall may miss the just-completed turn)", + timeout, dropped, + ) return False - time.sleep(0.05) + time.sleep(self._RETAIN_OP_POLL_INTERVAL_S) def _writer_loop(self) -> None: """Drain the retain queue serially. Exits on sentinel. diff --git a/tests/plugins/memory/test_hindsight_provider.py b/tests/plugins/memory/test_hindsight_provider.py index fef2bf3e7f5a..e3ba1b8a55ba 100644 --- a/tests/plugins/memory/test_hindsight_provider.py +++ b/tests/plugins/memory/test_hindsight_provider.py @@ -649,6 +649,38 @@ async def _recall(**kwargs): assert order == ["recall"], "prefetch should recall after the timeout" assert elapsed < 3.0, "prefetch must not block well past the drain budget" + def test_timed_out_ops_are_dropped_not_repolled(self, provider_with_config): + """Ops unresolved at deadline must be EVICTED so a permanently failing + status endpoint can't make every later prefetch re-burn the full + timeout on a growing pending set (unbounded session-wide degradation + + reply-path join penalty).""" + p = provider_with_config(prefetch_retain_drain_timeout=0.3) + p._client = self._client_with_ops(["pending"]) # never completes + p._client.arecall = AsyncMock( + return_value=SimpleNamespace(results=[SimpleNamespace(text="m")]) + ) + + p.sync_turn("hello", "world") + p._retain_queue.join() + assert p._pending_retain_ops, "op should be tracked before the wait" + + # First prefetch burns the budget and must DROP the wedged op. + p.queue_prefetch("q1") + if p._prefetch_thread: + p._prefetch_thread.join(timeout=5.0) + assert p._pending_retain_ops == set(), ( + "unresolved ops must be evicted at deadline, not retained" + ) + + # A later prefetch with nothing pending must be near-instant. + start = time.monotonic() + p.queue_prefetch("q2") + if p._prefetch_thread: + p._prefetch_thread.join(timeout=5.0) + assert time.monotonic() - start < 0.25, ( + "second prefetch re-polled dropped ops — eviction regressed" + ) + def test_operation_notfound_treated_as_complete(self, provider): """A NotFound (completed+evicted) op is treated as done, not pending.""" from hindsight_client_api.exceptions import NotFoundException From 74b28c8910ebb369ef2a0d278cd28c934d533e61 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 2 Aug 2026 00:56:31 -0700 Subject: [PATCH 45/69] fix(secrets): scope memory plugin credential reads (hindsight/honcho/supermemory/mem0/retaindb) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Route HINDSIGHT_API_KEY/HINDSIGHT_LLM_API_KEY, HONCHO_API_KEY, SUPERMEMORY_API_KEY, MEM0_API_KEY and RETAINDB_API_KEY reads through agent.secret_scope.get_secret so multiplexed turns resolve the active profile's key instead of the process environment. Also guard supermemory post_setup's os.environ write on not is_multiplex_active() — writing one profile's key into the process-global env pollutes sibling profiles; the single-profile convenience path is unchanged. --- plugins/memory/hindsight/__init__.py | 14 ++++++++------ plugins/memory/honcho/client.py | 5 +++-- plugins/memory/mem0/__init__.py | 3 ++- plugins/memory/retaindb/__init__.py | 5 +++-- plugins/memory/supermemory/__init__.py | 17 +++++++++++++---- 5 files changed, 29 insertions(+), 15 deletions(-) diff --git a/plugins/memory/hindsight/__init__.py b/plugins/memory/hindsight/__init__.py index 8c93df87f6c3..086bb1239bcb 100644 --- a/plugins/memory/hindsight/__init__.py +++ b/plugins/memory/hindsight/__init__.py @@ -44,6 +44,8 @@ from datetime import datetime, timezone from typing import Any, Dict, List +from agent.secret_scope import get_secret + from agent.memory_provider import MemoryProvider from hermes_constants import get_hermes_home from tools.registry import tool_error @@ -384,7 +386,7 @@ def _load_config() -> dict: return { "mode": os.environ.get("HINDSIGHT_MODE", "cloud"), - "apiKey": os.environ.get("HINDSIGHT_API_KEY", ""), + "apiKey": get_secret("HINDSIGHT_API_KEY", ""), "timeout": _parse_int_setting(os.environ.get("HINDSIGHT_TIMEOUT"), _DEFAULT_TIMEOUT), "idle_timeout": _parse_int_setting(os.environ.get("HINDSIGHT_IDLE_TIMEOUT"), _DEFAULT_IDLE_TIMEOUT), "retain_tags": os.environ.get("HINDSIGHT_RETAIN_TAGS", ""), @@ -523,7 +525,7 @@ def _build_embedded_profile_env(config: dict[str, Any], *, llm_api_key: str | No current_key = ( config.get("llmApiKey") or config.get("llm_api_key") - or os.environ.get("HINDSIGHT_LLM_API_KEY", "") + or get_secret("HINDSIGHT_LLM_API_KEY", "") ) current_provider = config.get("llm_provider", "") @@ -813,7 +815,7 @@ def is_available(self) -> bool: has_key = bool( cfg.get("apiKey") or cfg.get("api_key") - or os.environ.get("HINDSIGHT_API_KEY", "") + or get_secret("HINDSIGHT_API_KEY", "") ) has_url = bool(cfg.get("api_url") or os.environ.get("HINDSIGHT_API_URL", "")) return has_key or has_url @@ -922,7 +924,7 @@ def post_setup(self, hermes_home: str, config: dict) -> None: # Step 3: Mode-specific config if mode == "cloud": print("\n Get your API key at https://ui.hindsight.vectorize.io\n") - existing_key = os.environ.get("HINDSIGHT_API_KEY", "") + existing_key = get_secret("HINDSIGHT_API_KEY", "") or "" if existing_key: masked = f"...{existing_key[-4:]}" if len(existing_key) > 4 else "set" sys.stdout.write(f" API key (current: {masked}, blank to keep): ") @@ -1123,7 +1125,7 @@ def _get_client(self): kwargs = dict( profile=self._config.get("profile", "hermes"), llm_provider=llm_provider, - llm_api_key=self._config.get("llmApiKey") or self._config.get("llm_api_key") or os.environ.get("HINDSIGHT_LLM_API_KEY", ""), + llm_api_key=self._config.get("llmApiKey") or self._config.get("llm_api_key") or get_secret("HINDSIGHT_LLM_API_KEY", ""), llm_model=self._config.get("llm_model", ""), ) if self._llm_base_url: @@ -1522,7 +1524,7 @@ def initialize(self, session_id: str, **kwargs) -> None: ) self._mode = "disabled" return - self._api_key = self._config.get("apiKey") or self._config.get("api_key") or os.environ.get("HINDSIGHT_API_KEY", "") + self._api_key = self._config.get("apiKey") or self._config.get("api_key") or get_secret("HINDSIGHT_API_KEY", "") default_url = _DEFAULT_LOCAL_URL if self._mode in {"local_embedded", "local_external"} else _DEFAULT_API_URL self._api_url = self._config.get("api_url") or os.environ.get("HINDSIGHT_API_URL", default_url) self._llm_base_url = self._config.get("llm_base_url", "") diff --git a/plugins/memory/honcho/client.py b/plugins/memory/honcho/client.py index cc1da55ffaf4..1945a38a15fe 100644 --- a/plugins/memory/honcho/client.py +++ b/plugins/memory/honcho/client.py @@ -22,6 +22,7 @@ from pathlib import Path from urllib.parse import urlparse +from agent.secret_scope import get_secret from hermes_constants import get_hermes_home from hermes_cli.profiles import _get_default_hermes_home from plugins.plugin_utils import SingletonSlot @@ -469,7 +470,7 @@ def from_env( ) -> HonchoClientConfig: """Create config from environment variables (fallback).""" resolved_host = host or resolve_active_host() - api_key = os.environ.get("HONCHO_API_KEY") + api_key = get_secret("HONCHO_API_KEY") base_url = os.environ.get("HONCHO_BASE_URL", "").strip() or None timeout = _resolve_optional_float(os.environ.get("HONCHO_TIMEOUT")) return cls( @@ -525,7 +526,7 @@ def from_global_config( api_key = ( host_block.get("apiKey") or raw.get("apiKey") - or os.environ.get("HONCHO_API_KEY") + or get_secret("HONCHO_API_KEY") ) environment = ( diff --git a/plugins/memory/mem0/__init__.py b/plugins/memory/mem0/__init__.py index b060a7bf7a8a..ea871c44cc41 100644 --- a/plugins/memory/mem0/__init__.py +++ b/plugins/memory/mem0/__init__.py @@ -41,6 +41,7 @@ from typing import Any, Dict, List from agent.memory_provider import MemoryProvider +from agent.secret_scope import get_secret from tools.registry import tool_error logger = logging.getLogger(__name__) @@ -85,7 +86,7 @@ def _load_config() -> dict: config = { "mode": os.environ.get("MEM0_MODE", "platform"), - "api_key": os.environ.get("MEM0_API_KEY", ""), + "api_key": get_secret("MEM0_API_KEY", ""), "host": os.environ.get("MEM0_HOST", ""), "agent_id": os.environ.get("MEM0_AGENT_ID", "hermes"), "oss": {}, diff --git a/plugins/memory/retaindb/__init__.py b/plugins/memory/retaindb/__init__.py index a777ccbd55ee..13ad0331501b 100644 --- a/plugins/memory/retaindb/__init__.py +++ b/plugins/memory/retaindb/__init__.py @@ -34,6 +34,7 @@ from urllib.parse import quote from agent.memory_provider import MemoryProvider +from agent.secret_scope import get_secret from agent.file_safety import raise_if_read_blocked from tools.registry import tool_error @@ -476,7 +477,7 @@ def name(self) -> str: return "retaindb" def is_available(self) -> bool: - return bool(os.environ.get("RETAINDB_API_KEY")) + return bool(get_secret("RETAINDB_API_KEY")) def get_config_schema(self) -> List[Dict[str, Any]]: return [ @@ -488,7 +489,7 @@ def get_config_schema(self) -> List[Dict[str, Any]]: # ── Lifecycle ────────────────────────────────────────────────────────── def initialize(self, session_id: str, **kwargs) -> None: - api_key = os.environ.get("RETAINDB_API_KEY", "") + api_key = get_secret("RETAINDB_API_KEY", "") or "" base_url = re.sub(r"/+$", "", os.environ.get("RETAINDB_BASE_URL", _DEFAULT_BASE_URL)) # Project resolution: RETAINDB_PROJECT > hermes- > "default" diff --git a/plugins/memory/supermemory/__init__.py b/plugins/memory/supermemory/__init__.py index 2112b5095d76..7d737fd36774 100644 --- a/plugins/memory/supermemory/__init__.py +++ b/plugins/memory/supermemory/__init__.py @@ -18,6 +18,7 @@ from typing import Any, Dict, List, Optional from agent.memory_provider import MemoryProvider +from agent.secret_scope import get_secret, is_multiplex_active from tools.registry import tool_error logger = logging.getLogger(__name__) @@ -572,7 +573,7 @@ def is_available(self) -> bool: # Docker venv the package isn't present until ensure() runs, but # ensure() only runs once the provider is loaded — which this gates. # Mirrors honcho/mem0, which check config only. No network calls. - return bool(os.environ.get("SUPERMEMORY_API_KEY", "")) + return bool(get_secret("SUPERMEMORY_API_KEY", "")) def get_config_schema(self): # Only prompt for the API key during `hermes memory setup`. @@ -595,7 +596,7 @@ def get_status_config(self, provider_config: dict) -> dict: del provider_config hermes_home = str(get_hermes_home()) - api_key = os.environ.get("SUPERMEMORY_API_KEY", "") + api_key = get_secret("SUPERMEMORY_API_KEY", "") or "" status = _probe_supermemory_connection(api_key, hermes_home) return {"summary": _format_connection_summary(status)} @@ -630,7 +631,15 @@ def post_setup(self, hermes_home: str, config: dict) -> None: # Make the freshly-entered key visible to the connection probe below. # (Checks the VALUE of SUPERMEMORY_API_KEY, not whether the key string # happens to name some unrelated env var.) - if api_key and os.environ.get("SUPERMEMORY_API_KEY") != api_key: + # Single-profile convenience only: never write a profile's key into + # the process-global environ under a multiplexed gateway — sibling + # profiles' turns (and any subprocess spawned with env=os.environ) + # would inherit it. + if ( + api_key + and not is_multiplex_active() + and os.environ.get("SUPERMEMORY_API_KEY") != api_key + ): os.environ["SUPERMEMORY_API_KEY"] = api_key status = _probe_supermemory_connection(api_key, hermes_home) @@ -647,7 +656,7 @@ def initialize(self, session_id: str, **kwargs) -> None: self._session_id = session_id self._turn_count = 0 self._config = _load_supermemory_config(self._hermes_home) - self._api_key = os.environ.get("SUPERMEMORY_API_KEY", "") + self._api_key = get_secret("SUPERMEMORY_API_KEY", "") or "" # Resolve container tag: env var > config > default. # Supports {identity} template for profile-scoped containers. From a23ede5569176d14b80f6411732d181c0fc80261 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 2 Aug 2026 00:56:31 -0700 Subject: [PATCH 46/69] fix(secrets): scope image_gen plugin credential reads (openai/deepinfra/krea) OPENAI_API_KEY, DEEPINFRA_API_KEY and KREA_API_KEY now resolve via get_secret; the OpenAI client is constructed with the scoped key explicitly instead of relying on the SDK's implicit environ read. --- plugins/image_gen/deepinfra/__init__.py | 5 +++-- plugins/image_gen/krea/__init__.py | 7 ++++--- plugins/image_gen/openai/__init__.py | 8 +++++--- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/plugins/image_gen/deepinfra/__init__.py b/plugins/image_gen/deepinfra/__init__.py index 1ef5dcfb6c7e..90754b369f76 100644 --- a/plugins/image_gen/deepinfra/__init__.py +++ b/plugins/image_gen/deepinfra/__init__.py @@ -31,6 +31,7 @@ import os from typing import Any, Dict, List, Optional +from agent.secret_scope import get_secret from agent.image_gen_provider import ( DEFAULT_ASPECT_RATIO, ImageGenProvider, @@ -138,7 +139,7 @@ def display_name(self) -> str: return "DeepInfra" def is_available(self) -> bool: - return bool(os.environ.get("DEEPINFRA_API_KEY", "").strip()) + return bool((get_secret("DEEPINFRA_API_KEY", "") or "").strip()) def list_models(self) -> List[Dict[str, Any]]: live = _live_models() @@ -199,7 +200,7 @@ def generate( aspect_ratio=aspect, ) - api_key = os.environ.get("DEEPINFRA_API_KEY", "").strip() + api_key = (get_secret("DEEPINFRA_API_KEY", "") or "").strip() if not api_key: return error_response( error=( diff --git a/plugins/image_gen/krea/__init__.py b/plugins/image_gen/krea/__init__.py index d7b260667aa6..64c53c447f17 100644 --- a/plugins/image_gen/krea/__init__.py +++ b/plugins/image_gen/krea/__init__.py @@ -30,6 +30,7 @@ import requests +from agent.secret_scope import get_secret from agent.image_gen_provider import ( DEFAULT_ASPECT_RATIO, ImageGenProvider, @@ -177,7 +178,7 @@ def _resolve_managed_krea_gateway(): logger.debug("Managed Krea gateway resolution unavailable: %s", exc) return None - if os.environ.get("KREA_API_KEY") and not prefers_gateway("image_gen"): + if get_secret("KREA_API_KEY") and not prefers_gateway("image_gen"): return None try: @@ -233,7 +234,7 @@ def is_available(self) -> bool: # Available with a direct Krea key OR via the managed Nous gateway # (Nous Subscription), so portal users with no Krea key can still # reach Krea 2 through the gateway. - return bool(os.environ.get("KREA_API_KEY")) or _managed_krea_gateway_ready() + return bool(get_secret("KREA_API_KEY")) or _managed_krea_gateway_ready() def list_models(self) -> List[Dict[str, Any]]: return [ @@ -338,7 +339,7 @@ def generate( auth_token = managed.nous_user_token else: base_url = BASE_URL - auth_token = os.environ.get("KREA_API_KEY") + auth_token = get_secret("KREA_API_KEY") if not auth_token: return error_response( error=( diff --git a/plugins/image_gen/openai/__init__.py b/plugins/image_gen/openai/__init__.py index cfa9e42c908f..6fc4bc58e04b 100644 --- a/plugins/image_gen/openai/__init__.py +++ b/plugins/image_gen/openai/__init__.py @@ -27,6 +27,7 @@ import os from typing import Any, Dict, List, Optional, Tuple +from agent.secret_scope import get_secret from agent.image_gen_provider import ( DEFAULT_ASPECT_RATIO, ImageGenProvider, @@ -173,7 +174,7 @@ def display_name(self) -> str: return "OpenAI" def is_available(self) -> bool: - if not os.environ.get("OPENAI_API_KEY"): + if not get_secret("OPENAI_API_KEY"): return False try: import openai # noqa: F401 @@ -235,7 +236,8 @@ def generate( aspect_ratio=aspect, ) - if not os.environ.get("OPENAI_API_KEY"): + api_key = get_secret("OPENAI_API_KEY") + if not api_key: return error_response( error=( "OPENAI_API_KEY not set. Run `hermes tools` → Image " @@ -270,7 +272,7 @@ def generate( is_edit = bool(sources) modality = "image" if is_edit else "text" - client = openai.OpenAI() + client = openai.OpenAI(api_key=api_key) if is_edit: # images.edit() expects file-like objects. Download/read each From 2438305a220055196f6091c78f2ed4f4e4f6baa0 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 2 Aug 2026 00:56:31 -0700 Subject: [PATCH 47/69] fix(secrets): scope browser plugin credential reads (browser_use/browserbase/firecrawl) --- plugins/browser/browser_use/provider.py | 3 ++- plugins/browser/browserbase/provider.py | 5 +++-- plugins/browser/firecrawl/provider.py | 5 +++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/plugins/browser/browser_use/provider.py b/plugins/browser/browser_use/provider.py index e098cf814a65..a10d9a361363 100644 --- a/plugins/browser/browser_use/provider.py +++ b/plugins/browser/browser_use/provider.py @@ -37,6 +37,7 @@ import requests from agent.browser_provider import BrowserProvider +from agent.secret_scope import get_secret logger = logging.getLogger(__name__) @@ -137,7 +138,7 @@ def _get_config_or_none(self, *, refresh_token: bool = True) -> Optional[Dict[st # Direct API key wins unless the user has explicitly opted into the # managed Nous gateway via ``tool_gateway.browser: gateway``. - api_key = os.environ.get("BROWSER_USE_API_KEY") + api_key = get_secret("BROWSER_USE_API_KEY") if api_key and not prefers_gateway("browser"): return { "api_key": api_key, diff --git a/plugins/browser/browserbase/provider.py b/plugins/browser/browserbase/provider.py index c828ae29b2da..04829f740f53 100644 --- a/plugins/browser/browserbase/provider.py +++ b/plugins/browser/browserbase/provider.py @@ -39,6 +39,7 @@ import requests from agent.browser_provider import BrowserProvider +from agent.secret_scope import get_secret logger = logging.getLogger(__name__) @@ -66,8 +67,8 @@ def is_available(self) -> bool: # ------------------------------------------------------------------ def _get_config_or_none(self) -> Optional[Dict[str, Any]]: - api_key = os.environ.get("BROWSERBASE_API_KEY") - project_id = os.environ.get("BROWSERBASE_PROJECT_ID") + api_key = get_secret("BROWSERBASE_API_KEY") + project_id = get_secret("BROWSERBASE_PROJECT_ID") if api_key and project_id: return { "api_key": api_key, diff --git a/plugins/browser/firecrawl/provider.py b/plugins/browser/firecrawl/provider.py index 892cd6c1dc8f..2177cd96e2ff 100644 --- a/plugins/browser/firecrawl/provider.py +++ b/plugins/browser/firecrawl/provider.py @@ -34,6 +34,7 @@ import requests from agent.browser_provider import BrowserProvider +from agent.secret_scope import get_secret logger = logging.getLogger(__name__) @@ -56,7 +57,7 @@ def display_name(self) -> str: return "Firecrawl" def is_available(self) -> bool: - return bool(os.environ.get("FIRECRAWL_API_KEY")) + return bool(get_secret("FIRECRAWL_API_KEY")) # ------------------------------------------------------------------ # Session lifecycle @@ -66,7 +67,7 @@ def _api_url(self) -> str: return os.environ.get("FIRECRAWL_API_URL", _BASE_URL) def _headers(self) -> Dict[str, str]: - api_key = os.environ.get("FIRECRAWL_API_KEY") + api_key = get_secret("FIRECRAWL_API_KEY") if not api_key: raise ValueError( "FIRECRAWL_API_KEY environment variable is required. " From ebd61ce5ac930c6a73e182556be8930521dab833 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 2 Aug 2026 00:56:31 -0700 Subject: [PATCH 48/69] fix(secrets): scope tier-3 credential reads (teams_pipeline, deepinfra models, FAL/XAI/VERCEL/DAYTONA/GITHUB presence, HERMES_API_KEY display) NOTION_API_KEY/LINEAR_API_KEY defaults, DEEPINFRA_API_KEY model gate, FAL_KEY via the file's own _scoped_credential helper, XAI/VERCEL/DAYTONA presence checks, GITHUB_TOKEN/GH_TOKEN + GitHub App creds in tirith/skills_hub, and the masked HERMES_API_KEY display in tui_gateway config.show all route through get_secret. --- hermes_cli/tools_config.py | 6 +++++- plugins/model-providers/deepinfra/__init__.py | 4 ++-- plugins/teams_pipeline/pipeline.py | 6 ++++-- tools/skills_hub.py | 12 +++++++----- tools/terminal_tool.py | 13 ++++++++----- tools/tirith_security.py | 3 ++- tools/tool_backend_helpers.py | 2 +- tools/xai_http.py | 10 ++++++++-- tui_gateway/methods_tools.py | 4 +++- 9 files changed, 40 insertions(+), 20 deletions(-) diff --git a/hermes_cli/tools_config.py b/hermes_cli/tools_config.py index 257eb0b72f6b..4d3a8c47530a 100644 --- a/hermes_cli/tools_config.py +++ b/hermes_cli/tools_config.py @@ -189,7 +189,11 @@ def _xai_credentials_present() -> bool: return True except Exception: pass - return bool(str(os.environ.get("XAI_API_KEY") or "").strip()) + try: + from agent.secret_scope import get_secret + except ImportError: # pragma: no cover — secret_scope is in-repo + return bool(str(os.environ.get("XAI_API_KEY") or "").strip()) + return bool(str(get_secret("XAI_API_KEY") or "").strip()) def _homeassistant_credentials_present() -> bool: diff --git a/plugins/model-providers/deepinfra/__init__.py b/plugins/model-providers/deepinfra/__init__.py index afa06548db16..57c6a9c4ce0c 100644 --- a/plugins/model-providers/deepinfra/__init__.py +++ b/plugins/model-providers/deepinfra/__init__.py @@ -29,9 +29,9 @@ def default_vision_model(self): # type: ignore[override] ``vision`` capability) so an image-gen/edit model that merely carries a ``vision`` tag can't be picked as a chat-completions vision backend. """ - import os + from agent.secret_scope import get_secret - if not (os.environ.get("DEEPINFRA_API_KEY") or "").strip(): + if not (get_secret("DEEPINFRA_API_KEY") or "").strip(): return None try: from hermes_cli.models import _fetch_deepinfra_models_by_tag diff --git a/plugins/teams_pipeline/pipeline.py b/plugins/teams_pipeline/pipeline.py index a4c600b11ec3..38d66717bf3c 100644 --- a/plugins/teams_pipeline/pipeline.py +++ b/plugins/teams_pipeline/pipeline.py @@ -15,6 +15,8 @@ import httpx +from agent.secret_scope import get_secret + from agent.auxiliary_client import async_call_llm, extract_content_or_reasoning from hermes_constants import get_hermes_home from plugins.teams_pipeline.meetings import ( @@ -109,7 +111,7 @@ class NotionWriter: API_VERSION = "2025-09-03" def __init__(self, *, api_key: str | None = None, transport: httpx.AsyncBaseTransport | None = None) -> None: - self.api_key = (api_key or os.getenv("NOTION_API_KEY", "")).strip() + self.api_key = (api_key or get_secret("NOTION_API_KEY", "") or "").strip() self._transport = transport async def write_summary( @@ -205,7 +207,7 @@ class LinearWriter: API_URL = "https://api.linear.app/graphql" def __init__(self, *, api_key: str | None = None, transport: httpx.AsyncBaseTransport | None = None) -> None: - self.api_key = (api_key or os.getenv("LINEAR_API_KEY", "")).strip() + self.api_key = (api_key or get_secret("LINEAR_API_KEY", "") or "").strip() self._transport = transport async def write_summary( diff --git a/tools/skills_hub.py b/tools/skills_hub.py index a193895ca5cf..56964c132561 100644 --- a/tools/skills_hub.py +++ b/tools/skills_hub.py @@ -382,8 +382,9 @@ def _resolve_token(self) -> Optional[str]: if self._cached_method != "github-app" or time.time() < self._app_token_expiry: return self._cached_token - # 1. Environment variable - token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") + # 1. Environment variable (profile-scoped under a multiplexed gateway) + from agent.secret_scope import get_secret + token = get_secret("GITHUB_TOKEN") or get_secret("GH_TOKEN") if token: self._cached_token = token self._cached_method = "pat" @@ -424,9 +425,10 @@ def _try_gh_cli(self) -> Optional[str]: def _try_github_app(self) -> Optional[str]: """Try GitHub App JWT authentication if credentials are configured.""" - app_id = os.environ.get("GITHUB_APP_ID") - key_path = os.environ.get("GITHUB_APP_PRIVATE_KEY_PATH") - installation_id = os.environ.get("GITHUB_APP_INSTALLATION_ID") + from agent.secret_scope import get_secret + app_id = get_secret("GITHUB_APP_ID") + key_path = get_secret("GITHUB_APP_PRIVATE_KEY_PATH") + installation_id = get_secret("GITHUB_APP_INSTALLATION_ID") if not all([app_id, key_path, installation_id]): return None diff --git a/tools/terminal_tool.py b/tools/terminal_tool.py index 244449ce97bc..6572174819af 100644 --- a/tools/terminal_tool.py +++ b/tools/terminal_tool.py @@ -159,10 +159,12 @@ def _check_vercel_sandbox_requirements(config: dict[str, Any]) -> bool: ) return False - has_oidc = bool(os.getenv("VERCEL_OIDC_TOKEN")) - has_token = bool(os.getenv("VERCEL_TOKEN")) - has_project = bool(os.getenv("VERCEL_PROJECT_ID")) - has_team = bool(os.getenv("VERCEL_TEAM_ID")) + from agent.secret_scope import get_secret + + has_oidc = bool(get_secret("VERCEL_OIDC_TOKEN")) + has_token = bool(get_secret("VERCEL_TOKEN")) + has_project = bool(get_secret("VERCEL_PROJECT_ID")) + has_team = bool(get_secret("VERCEL_TEAM_ID")) if has_oidc: return True @@ -3189,7 +3191,8 @@ def check_terminal_requirements() -> bool: elif env_type == "daytona": from daytona import Daytona # noqa: F401 — SDK presence check - return os.getenv("DAYTONA_API_KEY") is not None + from agent.secret_scope import get_secret + return get_secret("DAYTONA_API_KEY") is not None else: logger.error( diff --git a/tools/tirith_security.py b/tools/tirith_security.py index 2ea2eec749d9..a284c6d4007c 100644 --- a/tools/tirith_security.py +++ b/tools/tirith_security.py @@ -283,7 +283,8 @@ def is_platform_supported() -> bool: def _download_file(url: str, dest: str, timeout: int = 10): """Download a URL to a local file.""" req = urllib.request.Request(url) - token = os.getenv("GITHUB_TOKEN") + from agent.secret_scope import get_secret + token = get_secret("GITHUB_TOKEN") if token: req.add_header("Authorization", f"token {token}") with urllib.request.urlopen(req, timeout=timeout) as resp, open(dest, "wb") as f: diff --git a/tools/tool_backend_helpers.py b/tools/tool_backend_helpers.py index de1fec7e8c5b..b00814c0ad23 100644 --- a/tools/tool_backend_helpers.py +++ b/tools/tool_backend_helpers.py @@ -298,7 +298,7 @@ def fal_key_is_configured() -> bool: checks and CLI setup-time checks agree. A whitespace-only value is treated as unset everywhere. """ - value = os.getenv("FAL_KEY") + value = _scoped_credential("FAL_KEY") or None if value is None: # Fall back to the .env file for CLI paths that may run before # dotenv is loaded into os.environ. diff --git a/tools/xai_http.py b/tools/xai_http.py index 1e4f72a7fb0a..5350a27d76f7 100644 --- a/tools/xai_http.py +++ b/tools/xai_http.py @@ -36,8 +36,14 @@ def has_xai_credentials() -> bool: other availability scans. Truthful refresh + expiry handling happens in ``search()`` (or whichever caller actually makes the request). """ - if os.environ.get("XAI_API_KEY", "").strip(): - return True + try: + from agent.secret_scope import get_secret + except ImportError: # pragma: no cover — secret_scope is in-repo + if os.environ.get("XAI_API_KEY", "").strip(): + return True + else: + if (get_secret("XAI_API_KEY", "") or "").strip(): + return True try: from hermes_constants import get_hermes_home diff --git a/tui_gateway/methods_tools.py b/tui_gateway/methods_tools.py index 53be0136bcc5..5896f1281c77 100644 --- a/tui_gateway/methods_tools.py +++ b/tui_gateway/methods_tools.py @@ -1384,7 +1384,9 @@ def _(rid, params: dict) -> dict: try: cfg = _load_cfg() model = _resolve_model() - api_key = os.environ.get("HERMES_API_KEY", "") or cfg.get("api_key", "") + from agent.secret_scope import get_secret + + api_key = get_secret("HERMES_API_KEY", "") or cfg.get("api_key", "") masked = f"****{api_key[-4:]}" if len(api_key) > 4 else "(not set)" base_url = os.environ.get("HERMES_BASE_URL", "") or cfg.get("base_url", "") From 99533f70b163a90f8e7cb61063beb4ea650180ee Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 2 Aug 2026 00:56:31 -0700 Subject: [PATCH 49/69] fix(secrets): resolve google_meet realtime key via secret scope at spawn time The detached meet_bot child inherits the process environment, not the parent's contextvar secret scope, so process_manager.start() now resolves HERMES_MEET_REALTIME_KEY/OPENAI_API_KEY through get_secret in the parent and passes it explicitly in the child env (spawn-wrap shape). Adds the consolidated per-family regression test file. --- plugins/google_meet/meet_bot.py | 4 + plugins/google_meet/process_manager.py | 16 ++ tests/test_secret_scope_plugin_families.py | 260 +++++++++++++++++++++ 3 files changed, 280 insertions(+) create mode 100644 tests/test_secret_scope_plugin_families.py diff --git a/plugins/google_meet/meet_bot.py b/plugins/google_meet/meet_bot.py index 0396d039490a..20745eb3de37 100644 --- a/plugins/google_meet/meet_bot.py +++ b/plugins/google_meet/meet_bot.py @@ -456,6 +456,10 @@ def run_bot() -> int: # noqa: C901 — orchestration, explicit branches realtime_model = os.environ.get("HERMES_MEET_REALTIME_MODEL", "gpt-realtime") realtime_voice = os.environ.get("HERMES_MEET_REALTIME_VOICE", "alloy") realtime_instructions = os.environ.get("HERMES_MEET_REALTIME_INSTRUCTIONS", "") + # HERMES_MEET_REALTIME_KEY is set explicitly by process_manager.start(), + # which resolves it through the parent's profile secret scope at spawn + # time. The bare OPENAI_API_KEY fallback only serves standalone + # `python -m plugins.google_meet.meet_bot` runs outside the gateway. realtime_api_key = os.environ.get("HERMES_MEET_REALTIME_KEY") or os.environ.get("OPENAI_API_KEY", "") if not url or not _is_safe_meet_url(url): diff --git a/plugins/google_meet/process_manager.py b/plugins/google_meet/process_manager.py index 0709c6a1f944..4b2576efb486 100644 --- a/plugins/google_meet/process_manager.py +++ b/plugins/google_meet/process_manager.py @@ -152,6 +152,22 @@ def start( env["HERMES_MEET_REALTIME_VOICE"] = realtime_voice if realtime_instructions: env["HERMES_MEET_REALTIME_INSTRUCTIONS"] = realtime_instructions + # Resolve the realtime key at SPAWN time, in the parent, where the + # profile secret scope (a contextvar) is still installed. The detached + # child inherits the process environment — NOT the scope — so under a + # multiplexed gateway an in-child os.environ read would see another + # profile's OPENAI_API_KEY (or nothing). Pass it explicitly instead; + # meet_bot checks HERMES_MEET_REALTIME_KEY before OPENAI_API_KEY. + if not realtime_api_key: + try: + from agent.secret_scope import get_secret + + realtime_api_key = ( + get_secret("HERMES_MEET_REALTIME_KEY") + or get_secret("OPENAI_API_KEY") + ) + except ImportError: # pragma: no cover — secret_scope is in-repo + pass if realtime_api_key: env["HERMES_MEET_REALTIME_KEY"] = realtime_api_key diff --git a/tests/test_secret_scope_plugin_families.py b/tests/test_secret_scope_plugin_families.py new file mode 100644 index 000000000000..25d093b7185a --- /dev/null +++ b/tests/test_secret_scope_plugin_families.py @@ -0,0 +1,260 @@ +"""Regression tests: plugin-family credential reads honor the profile secret scope. + +Class-closure follow-up to the profile secret-scope cluster (#76462). Memory, +image_gen, and browser plugins, plus a handful of tier-3 tool helpers, read +credentials straight from ``os.environ``. Under a multiplexed gateway the +process environment may hold ANOTHER profile's key (or none), so every +credential read must route through ``agent.secret_scope.get_secret`` and honor +its verdict — a scoped miss under multiplexing returns the default and must +NOT borrow from ``os.environ``. + +One representative test pair (scoped-wins / scoped-miss-no-borrow) per plugin +family, plus the two behavioral sites: + +* supermemory ``post_setup`` must not write a profile's key into the + process-global environ when multiplexing is active (sibling-profile + pollution). +* google_meet ``process_manager.start`` must resolve OPENAI_API_KEY through + the scope AT SPAWN TIME and pass it explicitly in the child environment — + the detached child inherits the process env, not the contextvar scope. +""" + +from __future__ import annotations + +import os +from typing import Any, Dict + +import pytest + +from agent.secret_scope import ( + reset_secret_scope, + set_multiplex_active, + set_secret_scope, +) + + +@pytest.fixture +def multiplex_scope(): + """Install a secret scope with multiplexing ON; restore state after.""" + + def _install(scope: Dict[str, str]): + set_multiplex_active(True) + token = set_secret_scope(scope) + return token + + tokens = [] + + def install(scope: Dict[str, str]): + tokens.append(_install(scope)) + + yield install + + for token in tokens: + reset_secret_scope(token) + set_multiplex_active(False) + + +# --------------------------------------------------------------------------- +# Family A — memory plugins +# --------------------------------------------------------------------------- + +class TestMemoryFamily: + def test_retaindb_scoped_key_wins(self, multiplex_scope, monkeypatch): + monkeypatch.setenv("RETAINDB_API_KEY", "env-other-profile") + multiplex_scope({"RETAINDB_API_KEY": "scoped-key"}) + + from plugins.memory.retaindb import RetainDBMemoryProvider + + assert RetainDBMemoryProvider().is_available() is True + + def test_retaindb_scoped_miss_does_not_borrow_environ( + self, multiplex_scope, monkeypatch + ): + # Env holds another profile's key; the active profile's scope has none. + monkeypatch.setenv("RETAINDB_API_KEY", "env-other-profile") + multiplex_scope({}) + + from plugins.memory.retaindb import RetainDBMemoryProvider + + assert RetainDBMemoryProvider().is_available() is False + + def test_supermemory_scoped_miss_does_not_borrow_environ( + self, multiplex_scope, monkeypatch + ): + monkeypatch.setenv("SUPERMEMORY_API_KEY", "env-other-profile") + multiplex_scope({}) + + from plugins.memory.supermemory import SupermemoryMemoryProvider + + assert SupermemoryMemoryProvider().is_available() is False + + def test_supermemory_post_setup_no_environ_write_under_multiplex( + self, multiplex_scope, monkeypatch, tmp_path + ): + """post_setup must not pollute process env with a profile's key.""" + multiplex_scope({}) + monkeypatch.delenv("SUPERMEMORY_API_KEY", raising=False) + + import hermes_cli.config as cli_config + import hermes_cli.memory_setup as memory_setup + import plugins.memory.supermemory as sm + + monkeypatch.setattr(memory_setup, "_prompt", lambda *a, **k: "sm-fresh-key") + monkeypatch.setattr(memory_setup, "_write_env_vars", lambda *a, **k: None) + monkeypatch.setattr(cli_config, "save_config", lambda *a, **k: None) + monkeypatch.setattr( + sm, + "_probe_supermemory_connection", + lambda *a, **k: {"ok": True, "detail": "stub"}, + ) + monkeypatch.setattr(sm, "_format_connection_summary", lambda s: "stub") + + sm.SupermemoryMemoryProvider().post_setup(str(tmp_path), {}) + + assert "SUPERMEMORY_API_KEY" not in os.environ + + def test_supermemory_post_setup_environ_write_kept_single_profile( + self, monkeypatch, tmp_path + ): + """Single-profile (multiplex off): the convenience write still happens.""" + set_multiplex_active(False) + monkeypatch.delenv("SUPERMEMORY_API_KEY", raising=False) + + import hermes_cli.config as cli_config + import hermes_cli.memory_setup as memory_setup + import plugins.memory.supermemory as sm + + monkeypatch.setattr(memory_setup, "_prompt", lambda *a, **k: "sm-fresh-key") + monkeypatch.setattr(memory_setup, "_write_env_vars", lambda *a, **k: None) + monkeypatch.setattr(cli_config, "save_config", lambda *a, **k: None) + monkeypatch.setattr( + sm, + "_probe_supermemory_connection", + lambda *a, **k: {"ok": True, "detail": "stub"}, + ) + monkeypatch.setattr(sm, "_format_connection_summary", lambda s: "stub") + + try: + sm.SupermemoryMemoryProvider().post_setup(str(tmp_path), {}) + assert os.environ.get("SUPERMEMORY_API_KEY") == "sm-fresh-key" + finally: + os.environ.pop("SUPERMEMORY_API_KEY", None) + + +# --------------------------------------------------------------------------- +# Family B — image_gen plugins +# --------------------------------------------------------------------------- + +class TestImageGenFamily: + def test_deepinfra_scoped_key_wins(self, multiplex_scope, monkeypatch): + monkeypatch.delenv("DEEPINFRA_API_KEY", raising=False) + multiplex_scope({"DEEPINFRA_API_KEY": "scoped-key"}) + + from plugins.image_gen.deepinfra import DeepInfraImageGenProvider + + assert DeepInfraImageGenProvider().is_available() is True + + def test_deepinfra_scoped_miss_does_not_borrow_environ( + self, multiplex_scope, monkeypatch + ): + monkeypatch.setenv("DEEPINFRA_API_KEY", "env-other-profile") + multiplex_scope({}) + + from plugins.image_gen.deepinfra import DeepInfraImageGenProvider + + assert DeepInfraImageGenProvider().is_available() is False + + +# --------------------------------------------------------------------------- +# Family C — browser/web plugins +# --------------------------------------------------------------------------- + +class TestBrowserFamily: + def test_firecrawl_scoped_key_wins(self, multiplex_scope, monkeypatch): + monkeypatch.delenv("FIRECRAWL_API_KEY", raising=False) + multiplex_scope({"FIRECRAWL_API_KEY": "scoped-key"}) + + from plugins.browser.firecrawl.provider import FirecrawlBrowserProvider + + provider = FirecrawlBrowserProvider() + assert provider.is_available() is True + assert provider._headers()["Authorization"] == "Bearer scoped-key" + + def test_firecrawl_scoped_miss_does_not_borrow_environ( + self, multiplex_scope, monkeypatch + ): + monkeypatch.setenv("FIRECRAWL_API_KEY", "env-other-profile") + multiplex_scope({}) + + from plugins.browser.firecrawl.provider import FirecrawlBrowserProvider + + assert FirecrawlBrowserProvider().is_available() is False + + +# --------------------------------------------------------------------------- +# Family E — google_meet spawn-wrap +# --------------------------------------------------------------------------- + +class TestGoogleMeetSpawn: + def test_child_env_carries_scoped_openai_key( + self, multiplex_scope, monkeypatch, tmp_path + ): + """start() resolves the key from the scope and injects it explicitly.""" + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.delenv("HERMES_MEET_REALTIME_KEY", raising=False) + multiplex_scope({"OPENAI_API_KEY": "scoped-openai-key"}) + + import plugins.google_meet.process_manager as pm + + monkeypatch.setattr(pm, "_root", lambda: tmp_path) + + captured: Dict[str, Any] = {} + + class _FakeProc: + pid = 4242 + + def fake_popen(cmd, **kwargs): + captured["env"] = kwargs.get("env") + return _FakeProc() + + monkeypatch.setattr(pm.subprocess, "Popen", fake_popen) + + result = pm.start( + "https://meet.google.com/abc-defg-hij", + out_dir=tmp_path / "meeting", + mode="realtime", + ) + + assert result["ok"] is True + child_env = captured["env"] + # The scoped key crosses the process boundary explicitly, not via + # inherited os.environ (which had no key at all). + assert child_env["HERMES_MEET_REALTIME_KEY"] == "scoped-openai-key" + + def test_explicit_key_argument_still_wins( + self, multiplex_scope, monkeypatch, tmp_path + ): + multiplex_scope({"OPENAI_API_KEY": "scoped-openai-key"}) + + import plugins.google_meet.process_manager as pm + + monkeypatch.setattr(pm, "_root", lambda: tmp_path) + + captured: Dict[str, Any] = {} + + class _FakeProc: + pid = 4243 + + monkeypatch.setattr( + pm.subprocess, + "Popen", + lambda cmd, **kw: (captured.update(env=kw.get("env")), _FakeProc())[1], + ) + + pm.start( + "https://meet.google.com/abc-defg-hij", + out_dir=tmp_path / "meeting2", + realtime_api_key="explicit-key", + ) + + assert captured["env"]["HERMES_MEET_REALTIME_KEY"] == "explicit-key" From da43a8527b47f4cff6a1e6ab93eabb44febd435e Mon Sep 17 00:00:00 2001 From: Ryder Freeman Date: Fri, 17 Jul 2026 07:29:25 -0700 Subject: [PATCH 50/69] feat(mem): config-driven allocator trim with telemetry and lifecycle coverage Add config-driven glibc malloc_trim for long-lived Hermes processes: - hermes_cli/mem_trim.py: trim_memory() with configurable cooldown, RSS snapshot telemetry, and forced-trim INFO logging - gateway/run.py: periodic trim in gateway housekeeping loop - tui_gateway/server.py: trim in idle reaper (~every 5 min) - tui_gateway/slash_worker.py: trim on turn boundary - run_agent.py: force trim on agent close - hermes_cli/config.py: context.memory_trim config section (enabled, cooldown_seconds, log_every_n, info_log_min_delta_mb) CSA tier-4 reviewed (4 rounds, 0 HIGH/MEDIUM/CRITICAL remaining). Supersedes PR #63708 + #64591 with enhanced telemetry and gateway/slash_worker coverage. --- gateway/run.py | 16 ++ hermes_cli/config_defaults.py | 12 + hermes_cli/mem_trim.py | 238 ++++++++++++++++++ run_agent.py | 9 + .../gateway/test_memory_trim_housekeeping.py | 32 +++ tests/hermes_cli/test_mem_trim.py | 180 +++++++++++++ tests/test_tui_gateway_server.py | 110 ++++++++ tui_gateway/server.py | 30 +++ tui_gateway/slash_worker.py | 15 ++ 9 files changed, 642 insertions(+) create mode 100644 hermes_cli/mem_trim.py create mode 100644 tests/gateway/test_memory_trim_housekeeping.py create mode 100644 tests/hermes_cli/test_mem_trim.py diff --git a/gateway/run.py b/gateway/run.py index 2d870b2cd8cf..1a92da6bebfe 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -25318,6 +25318,7 @@ def _start_gateway_housekeeping(stop_event: threading.Event, adapters=None, loop PASTE_SWEEP_EVERY = 60 # ticks — once per hour CURATOR_EVERY = 60 # ticks — poll hourly (inner gate handles the real cadence) AUTO_ARCHIVE_EVERY = 60 # ticks — poll hourly (state_meta gate owns the real cadence) + MEMORY_TRIM_EVERY = 1 # shared helper cooldown bounds actual allocator work # Every platform media cache prunes on the same hourly cadence — one loop # over (name, cleanup_fn), not a copy-pasted try/except per cache. @@ -25426,6 +25427,21 @@ def _start_gateway_housekeeping(stop_event: threading.Event, adapters=None, loop except Exception as e: logger.debug("Auto-archive tick error: %s", e) + # This is the long-lived messaging-gateway counterpart to the TUI idle + # reaper. The helper is config-gated and rate-limited, so calling it on + # the 60s housekeeping cadence does not create a trim storm. + if tick_count % MEMORY_TRIM_EVERY == 0: + try: + from hermes_cli.mem_trim import trim_memory + + trim_memory(reason="messaging gateway housekeeping") + except Exception as exc: + logger.warning( + "gateway housekeeping memory trim failed: %s: %s", + type(exc).__name__, + exc, + ) + stop_event.wait(timeout=interval) logger.info("Gateway housekeeping stopped") diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py index 4545575db26c..3181da0a3692 100644 --- a/hermes_cli/config_defaults.py +++ b/hermes_cli/config_defaults.py @@ -1587,6 +1587,18 @@ # a plugin in plugins/context_engine// or ~/.hermes/plugins/. "context": { "engine": "compressor", + # Return freed glibc allocator pages after long-running agent/TUI + # cleanup boundaries. Unsupported platforms are safe no-ops. + "memory_trim": { + "enabled": True, + "cooldown_seconds": 60.0, + # Successful trim calls are INFO logged every Nth periodic call; + # force paths always log so process-close behavior is visible. + "log_every_n": 1, + # Suppress INFO logs only when a readable RSS change is smaller. + # 0 reports every successful configured trim. + "info_log_min_delta_mb": 0.0, + }, }, # Persistent memory -- bounded curated memory injected into system prompt diff --git a/hermes_cli/mem_trim.py b/hermes_cli/mem_trim.py new file mode 100644 index 000000000000..ff15bea2ecb0 --- /dev/null +++ b/hermes_cli/mem_trim.py @@ -0,0 +1,238 @@ +"""Rate-limited heap release for long-lived Hermes gateway processes. + +On Linux/glibc, ``malloc_trim(0)`` can return pages from freed Python/C +allocations to the OS. Other platforms and allocators are safe no-ops. +Behavior is configured under ``context.memory_trim`` in ``config.yaml``. +""" + +from __future__ import annotations + +import ctypes +import gc +import logging +import platform +import sys +import threading +import time +from collections.abc import Callable +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + +_DEFAULT_COOLDOWN_SECONDS = 60.0 +_DEFAULT_LOG_EVERY_N = 1 +_DEFAULT_INFO_LOG_MIN_DELTA_MB = 0.0 +_trim_lock = threading.Lock() +_last_trim_monotonic = 0.0 +_probe_done = False +_malloc_trim: Callable[[int], int] | None = None +_trim_call_count = 0 + + +def _config_settings() -> tuple[bool, float, int, float]: + """Return fail-open settings from the normal Hermes config path.""" + enabled = True + cooldown: Any = _DEFAULT_COOLDOWN_SECONDS + log_every_n: Any = _DEFAULT_LOG_EVERY_N + info_log_min_delta_mb: Any = _DEFAULT_INFO_LOG_MIN_DELTA_MB + try: + from hermes_cli.config import load_config + + config = load_config() or {} + context = config.get("context") if isinstance(config, dict) else None + settings = context.get("memory_trim") if isinstance(context, dict) else None + if isinstance(settings, dict): + configured_enabled = settings.get("enabled") + if isinstance(configured_enabled, bool): + enabled = configured_enabled + cooldown = settings.get("cooldown_seconds", _DEFAULT_COOLDOWN_SECONDS) + log_every_n = settings.get("log_every_n", _DEFAULT_LOG_EVERY_N) + info_log_min_delta_mb = settings.get( + "info_log_min_delta_mb", _DEFAULT_INFO_LOG_MIN_DELTA_MB + ) + except Exception: + pass + return ( + enabled, + _cooldown_seconds(cooldown), + _log_every_n(log_every_n), + _nonnegative_float(info_log_min_delta_mb, _DEFAULT_INFO_LOG_MIN_DELTA_MB), + ) + + +def _cooldown_seconds(value: Any) -> float: + if isinstance(value, bool): + return _DEFAULT_COOLDOWN_SECONDS + try: + return max(0.0, float(value)) + except (TypeError, ValueError): + return _DEFAULT_COOLDOWN_SECONDS + + +def _log_every_n(value: Any) -> int: + if isinstance(value, bool): + return _DEFAULT_LOG_EVERY_N + try: + return max(1, int(value)) + except (TypeError, ValueError): + return _DEFAULT_LOG_EVERY_N + + +def _nonnegative_float(value: Any, default: float) -> float: + if isinstance(value, bool): + return default + try: + return max(0.0, float(value)) + except (TypeError, ValueError): + return default + + +def _read_proc_status() -> str | None: + """Read Linux process status without making non-Linux callers special-case.""" + if sys.platform != "linux": + return None + try: + return Path("/proc/self/status").read_text(encoding="utf-8") + except OSError: + return None + + +def collect_memory_snapshot(history_bytes: int | None = None) -> dict[str, int | None]: + """Return lightweight process-memory telemetry for trim logs and canaries. + + ``VmRSS`` and ``RssAnon`` are Linux-only best effort fields. The helper is + intentionally dependency-free so allocation recovery never requires psutil. + """ + snapshot: dict[str, int | None] = { + "rss_kib": None, + "rss_anon_kib": None, + "thread_count": threading.active_count(), + } + status = _read_proc_status() + if status: + for line in status.splitlines(): + key, separator, raw_value = line.partition(":") + if not separator or key not in {"VmRSS", "RssAnon"}: + continue + value = raw_value.strip().split(maxsplit=1) + if value and value[0].isdigit(): + snapshot["rss_kib" if key == "VmRSS" else "rss_anon_kib"] = int(value[0]) + if isinstance(history_bytes, int) and history_bytes >= 0: + snapshot["history_bytes"] = history_bytes + return snapshot + + +def _should_log_trim( + *, force: bool, log_every_n: int, call_count: int, before: dict[str, int | None], + after: dict[str, int | None], info_log_min_delta_mb: float, +) -> bool: + # trim_memory calls this only after malloc_trim reported success. A forced + # successful trim is an explicit observability event, regardless of RSS. + if force: + return True + if not force and call_count % log_every_n: + return False + before_rss = before.get("rss_kib") + after_rss = after.get("rss_kib") + if before_rss is None or after_rss is None: + return True + return abs(after_rss - before_rss) >= info_log_min_delta_mb * 1024 + + +def _probe_glibc_malloc_trim() -> Callable[[int], int] | None: + """Resolve glibc's malloc_trim once; return None on unsupported systems.""" + global _malloc_trim, _probe_done + if _probe_done: + return _malloc_trim + _probe_done = True + if sys.platform != "linux": + return None + try: + if platform.libc_ver()[0].lower() != "glibc": + return None + libc = ctypes.CDLL(None) + trim = libc.malloc_trim + trim.argtypes = [ctypes.c_size_t] + trim.restype = ctypes.c_int + _malloc_trim = trim + except Exception as exc: + logger.debug("malloc_trim unavailable: %s", exc) + return _malloc_trim + + +def trim_memory( + *, + force: bool = False, + reason: str = "", + cooldown_seconds: float | None = None, +) -> bool: + """Collect cycles and ask glibc to release free heap pages. + + Returns ``True`` only when ``malloc_trim(0)`` ran and reported success. + Unsupported allocators, the config kill switch, cooldown suppression, and all + runtime errors return ``False`` without affecting the caller. + """ + ( + enabled, + configured_cooldown, + log_every_n, + info_log_min_delta_mb, + ) = _config_settings() + if not enabled: + return False + + global _last_trim_monotonic, _trim_call_count + with _trim_lock: + trim = _probe_glibc_malloc_trim() + if trim is None: + return False + now = time.monotonic() + cooldown = ( + configured_cooldown + if cooldown_seconds is None + else _cooldown_seconds(cooldown_seconds) + ) + if not force and _last_trim_monotonic and now - _last_trim_monotonic < cooldown: + return False + # Record the attempt before calling into libc so repeated failures do not + # turn every turn boundary into an expensive full collection. + _last_trim_monotonic = now + try: + before = collect_memory_snapshot() + started = time.perf_counter() + gc.collect() + trim_result = trim(0) + released = bool(trim_result) + after = collect_memory_snapshot() + duration_ms = (time.perf_counter() - started) * 1000 + _trim_call_count += 1 + if released and _should_log_trim( + force=force, + log_every_n=log_every_n, + call_count=_trim_call_count, + before=before, + after=after, + info_log_min_delta_mb=info_log_min_delta_mb, + ): + logger.info( + "memory trim: reason=%s malloc_trim=%s rss_kib=%s->%s " + "rss_anon_kib=%s->%s threads=%s duration_ms=%.1f", + reason or "cleanup", + trim_result, + before.get("rss_kib"), + after.get("rss_kib"), + before.get("rss_anon_kib"), + after.get("rss_anon_kib"), + after.get("thread_count"), + duration_ms, + ) + return released + except Exception as exc: + logger.warning( + "memory trim failed after %s: %s: %s", + reason or "cleanup", + type(exc).__name__, + exc, + ) + return False diff --git a/run_agent.py b/run_agent.py index 072730f23dee..0376c4b16cd5 100644 --- a/run_agent.py +++ b/run_agent.py @@ -4131,6 +4131,15 @@ def close(self) -> None: except Exception: pass + # The references above are now gone; on Linux/glibc, return their free + # heap pages immediately instead of retaining the process RSS high-water + # mark until exit. This helper is a safe no-op on other allocators. + try: + from hermes_cli.mem_trim import trim_memory + trim_memory(force=True, reason="agent close") + except Exception: + pass + # 8. Finalize the owned SQLite session row unless this agent is only a # temporary helper that deliberately handed session ownership forward # (manual compression helpers that rotate to a continuation session_id, diff --git a/tests/gateway/test_memory_trim_housekeeping.py b/tests/gateway/test_memory_trim_housekeeping.py new file mode 100644 index 000000000000..c8818f36b369 --- /dev/null +++ b/tests/gateway/test_memory_trim_housekeeping.py @@ -0,0 +1,32 @@ +"""Memory-trim coverage for the long-lived messaging gateway housekeeper.""" + +import gateway.run as gateway_run + + +class _OneTickStopEvent: + """Run one housekeeping tick without a sleep or background thread.""" + + def __init__(self): + self.waited = False + + def is_set(self): + return self.waited + + def wait(self, timeout=None): + self.waited = True + return True + + +def test_gateway_housekeeping_calls_periodic_memory_trim(monkeypatch): + import hermes_cli.mem_trim as mem_trim + + calls = [] + monkeypatch.setattr( + mem_trim, + "trim_memory", + lambda **kwargs: calls.append(kwargs) or True, + ) + + gateway_run._start_gateway_housekeeping(_OneTickStopEvent(), interval=0) + + assert calls == [{"reason": "messaging gateway housekeeping"}] diff --git a/tests/hermes_cli/test_mem_trim.py b/tests/hermes_cli/test_mem_trim.py new file mode 100644 index 000000000000..bf6936022fe4 --- /dev/null +++ b/tests/hermes_cli/test_mem_trim.py @@ -0,0 +1,180 @@ +"""Tests for the long-lived gateway heap-trim helper.""" + +from unittest.mock import Mock + +import pytest + +import hermes_cli.mem_trim as mem_trim + + +@pytest.fixture(autouse=True) +def _reset_trim_state(monkeypatch): + monkeypatch.setattr(mem_trim, "_last_trim_monotonic", 0.0) + monkeypatch.setattr(mem_trim, "_probe_done", True) + monkeypatch.setattr(mem_trim, "_malloc_trim", None) + monkeypatch.setattr(mem_trim, "_trim_call_count", 0) + + +def test_unsupported_allocator_is_noop_without_gc(monkeypatch): + collect = Mock() + monkeypatch.setattr(mem_trim.gc, "collect", collect) + + assert mem_trim.trim_memory(force=True, reason="test") is False + collect.assert_not_called() + + +def test_config_kill_switch_overrides_force_from_config_file(monkeypatch, tmp_path): + from hermes_constants import reset_hermes_home_override, set_hermes_home_override + + hermes_home = tmp_path / "hermes" + hermes_home.mkdir() + (hermes_home / "config.yaml").write_text( + "context:\n memory_trim:\n enabled: false\n", + encoding="utf-8", + ) + trim = Mock(return_value=1) + monkeypatch.setattr(mem_trim, "_malloc_trim", trim) + token = set_hermes_home_override(hermes_home) + + try: + assert mem_trim.trim_memory(force=True) is False + trim.assert_not_called() + finally: + reset_hermes_home_override(token) + + +def test_default_config_declares_memory_trim_controls(): + from hermes_cli.config import DEFAULT_CONFIG + + context = DEFAULT_CONFIG["context"] + assert isinstance(context, dict) + settings = context["memory_trim"] + assert isinstance(settings, dict) + assert isinstance(settings["enabled"], bool) + assert isinstance(settings["cooldown_seconds"], float) + + +def test_collect_memory_snapshot_parses_linux_proc_status(monkeypatch): + monkeypatch.setattr(mem_trim.sys, "platform", "linux") + monkeypatch.setattr( + mem_trim, + "_read_proc_status", + lambda: "Name:\tpython\nVmRSS:\t1234 kB\nRssAnon:\t567 kB\n", + ) + monkeypatch.setattr(mem_trim.threading, "active_count", lambda: 9) + + assert mem_trim.collect_memory_snapshot(history_bytes=42) == { + "rss_kib": 1234, + "rss_anon_kib": 567, + "thread_count": 9, + "history_bytes": 42, + } + + +def test_success_collects_then_trims(monkeypatch): + calls = [] + monkeypatch.setattr(mem_trim.gc, "collect", lambda: calls.append("gc")) + monkeypatch.setattr( + mem_trim, "_malloc_trim", lambda pad: calls.append(("trim", pad)) or 1 + ) + monkeypatch.setattr(mem_trim.time, "monotonic", lambda: 100.0) + + assert mem_trim.trim_memory(reason="turn", cooldown_seconds=60) is True + assert calls == ["gc", ("trim", 0)] + assert mem_trim._last_trim_monotonic == 100.0 + + +def test_success_logs_memory_snapshot_and_trim_result(monkeypatch, caplog): + monkeypatch.setattr(mem_trim.gc, "collect", lambda: None) + monkeypatch.setattr(mem_trim, "_malloc_trim", lambda _pad: 1) + monkeypatch.setattr(mem_trim.time, "monotonic", lambda: 100.0) + snapshots = iter( + ( + {"rss_kib": 4096, "rss_anon_kib": 3072, "thread_count": 3}, + {"rss_kib": 2048, "rss_anon_kib": 1024, "thread_count": 3}, + ) + ) + monkeypatch.setattr(mem_trim, "collect_memory_snapshot", lambda: next(snapshots)) + + with caplog.at_level("INFO", logger="hermes_cli.mem_trim"): + assert mem_trim.trim_memory(reason="test turn") is True + + assert "reason=test turn" in caplog.text + assert "malloc_trim=1" in caplog.text + assert "rss_kib=4096->2048" in caplog.text + + +def test_force_logs_even_when_periodic_log_sampling_skips(monkeypatch, caplog): + monkeypatch.setattr(mem_trim.gc, "collect", lambda: None) + monkeypatch.setattr(mem_trim, "_malloc_trim", lambda _pad: 1) + monkeypatch.setattr(mem_trim, "_config_settings", lambda: (True, 0.0, 99, 1.0)) + monkeypatch.setattr(mem_trim.time, "monotonic", lambda: 100.0) + monkeypatch.setattr( + mem_trim, + "collect_memory_snapshot", + lambda: {"rss_kib": 4096, "rss_anon_kib": 3072, "thread_count": 3}, + ) + + with caplog.at_level("INFO", logger="hermes_cli.mem_trim"): + assert mem_trim.trim_memory(reason="periodic") is True + assert mem_trim.trim_memory(force=True, reason="close") is True + + messages = [record.getMessage() for record in caplog.records] + assert not any("reason=periodic" in message for message in messages) + assert any("reason=close" in message for message in messages) + + +def test_cooldown_suppresses_repeated_collection(monkeypatch): + collect = Mock() + trim = Mock(return_value=1) + monkeypatch.setattr(mem_trim.gc, "collect", collect) + monkeypatch.setattr(mem_trim, "_malloc_trim", trim) + monkeypatch.setattr(mem_trim, "_last_trim_monotonic", 95.0) + monkeypatch.setattr(mem_trim.time, "monotonic", lambda: 100.0) + + assert mem_trim.trim_memory(cooldown_seconds=60) is False + collect.assert_not_called() + trim.assert_not_called() + assert mem_trim.trim_memory(force=True, cooldown_seconds=60) is True + + +def test_config_cooldown_controls_rate_limit(monkeypatch): + trim = Mock(return_value=1) + monkeypatch.setattr(mem_trim, "_malloc_trim", trim) + monkeypatch.setattr(mem_trim, "_last_trim_monotonic", 1.0) + monkeypatch.setattr(mem_trim.time, "monotonic", lambda: 100.0) + monkeypatch.setattr( + "hermes_cli.config.load_config", + lambda: { + "context": { + "memory_trim": {"enabled": True, "cooldown_seconds": 120.0} + } + }, + ) + + assert mem_trim.trim_memory() is False + trim.assert_not_called() + + +def test_legacy_environment_switch_does_not_control_behavior(monkeypatch): + trim = Mock(return_value=1) + monkeypatch.setattr(mem_trim, "_malloc_trim", trim) + monkeypatch.setenv("HERMES_DISABLE_MEMORY_TRIM", "1") + monkeypatch.setattr( + "hermes_cli.config.load_config", + lambda: {"context": {"memory_trim": {"enabled": True}}}, + ) + + assert mem_trim.trim_memory(force=True) is True + trim.assert_called_once_with(0) + + +def test_libc_failure_is_fail_open_and_rate_limited(monkeypatch): + trim = Mock(side_effect=RuntimeError("boom")) + monkeypatch.setattr(mem_trim, "_malloc_trim", trim) + monkeypatch.setattr(mem_trim.time, "monotonic", lambda: 100.0) + + assert mem_trim.trim_memory(reason="test", cooldown_seconds=60) is False + assert mem_trim._last_trim_monotonic == 100.0 + assert mem_trim.trim_memory(cooldown_seconds=60) is False + assert trim.call_count == 1 diff --git a/tests/test_tui_gateway_server.py b/tests/test_tui_gateway_server.py index d8900b80bacf..b2d8e918fa1f 100644 --- a/tests/test_tui_gateway_server.py +++ b/tests/test_tui_gateway_server.py @@ -14150,6 +14150,49 @@ def test_reap_idle_sessions_closes_only_evictable(monkeypatch): server._sessions.clear() +def test_reap_idle_sessions_calls_periodic_trim(monkeypatch): + """The idle reaper must call trim_memory every scan, even with no victims.""" + trim_calls = [] + monkeypatch.setattr(server, "_session_pending_kind", lambda sid: "") + monkeypatch.setattr(server, "_close_session_by_id", lambda *a, **k: None) + monkeypatch.setattr(server, "_enforce_session_cap", lambda: None) + monkeypatch.setattr(server, "_reclaim_orphaned_leases", lambda: None) + + # Patch the delayed import path: the function does + # `from hermes_cli.mem_trim import trim_memory` at call time. + import hermes_cli.mem_trim as mem_trim + + monkeypatch.setattr( + mem_trim, "trim_memory", + lambda **kw: trim_calls.append(kw.get("reason", "")) or True, + ) + + server._sessions.clear() + try: + server._reap_idle_sessions() + assert len(trim_calls) == 1 + assert trim_calls[0] == "idle reaper periodic trim" + finally: + server._sessions.clear() + + +def test_reap_idle_sessions_logs_trim_failure(monkeypatch, caplog): + monkeypatch.setattr(server, "_session_pending_kind", lambda sid: "") + monkeypatch.setattr(server, "_close_session_by_id", lambda *a, **k: None) + monkeypatch.setattr(server, "_enforce_session_cap", lambda: None) + monkeypatch.setattr(server, "_reclaim_orphaned_leases", lambda: None) + import hermes_cli.mem_trim as mem_trim + + monkeypatch.setattr(mem_trim, "trim_memory", lambda **_kw: (_ for _ in ()).throw(RuntimeError("boom"))) + server._sessions.clear() + try: + with caplog.at_level("WARNING", logger="tui_gateway.server"): + server._reap_idle_sessions() + assert "idle reaper memory trim failed: RuntimeError: boom" in caplog.text + finally: + server._sessions.clear() + + def test_session_create_records_ui_model_as_session_override(monkeypatch): """The desktop composer owns its model as plain UI state and ships it on session.create. The gateway must record it as a PER-SESSION override (built @@ -15536,3 +15579,70 @@ def start(self): assert captured.get("persist_user_message") == "hi" finally: server._sessions.pop("sid", None) + + +def test_prompt_submit_releases_old_history_before_heap_trim(monkeypatch): + """The trim boundary must not retain the just-pruned history snapshots.""" + observed = {} + cleanup_order = [] + + class _Agent: + def run_conversation( + self, prompt, conversation_history=None, stream_callback=None + ): + return { + "final_response": "reply", + "messages": [{"role": "assistant", "content": "reply"}], + } + + class _ImmediateThread: + def __init__(self, target=None, daemon=None): + self._target = target + + def start(self): + assert self._target is not None + self._target() + + def _inspect_trim_frame(**_kwargs): + import inspect + + cleanup_order.append("trim") + frame = inspect.currentframe() + assert frame is not None and frame.f_back is not None + caller_locals = frame.f_back.f_locals + observed["history"] = caller_locals.get("history") + observed["run_kwargs"] = caller_locals.get("run_kwargs") + + session = _session(agent=_Agent()) + session["profile_home"] = "/tmp/test-profile" + session["history"] = [ + {"role": "tool", "tool_call_id": "old", "content": "x" * 20_000} + ] + server._sessions["sid_trim"] = session + try: + monkeypatch.setattr(server.threading, "Thread", _ImmediateThread) + monkeypatch.setattr(server, "_get_usage", lambda _a: {}) + monkeypatch.setattr(server, "render_message", lambda _t, _c: "") + monkeypatch.setattr(server, "_emit", lambda *a: None) + monkeypatch.setattr(server, "set_hermes_home_override", lambda _home: object()) + monkeypatch.setattr( + server, + "reset_hermes_home_override", + lambda _token: cleanup_order.append("reset_home"), + ) + monkeypatch.setattr("hermes_cli.mem_trim.trim_memory", _inspect_trim_frame) + + resp = server.handle_request( + { + "id": "1", + "method": "prompt.submit", + "params": {"session_id": "sid_trim", "text": "hi"}, + } + ) + + assert resp is not None and resp.get("result") + assert not observed["history"] + assert not observed["run_kwargs"] + assert cleanup_order == ["trim", "reset_home"] + finally: + server._sessions.pop("sid_trim", None) diff --git a/tui_gateway/server.py b/tui_gateway/server.py index 029d8ae72367..f2e8fb647212 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -1054,6 +1054,19 @@ def _reap_idle_sessions() -> None: _close_session_by_id(sid, end_reason="idle_timeout") _enforce_session_cap() _reclaim_orphaned_leases() + # Periodic heap release for long-lived gateway processes. Even when no + # session is reaped, Python's generational GC rarely runs gen2 collection + # under steady-state allocation, and glibc retains freed pages as RSS. + # Calling trim_memory here ensures every reaper scan (default every 5 min) + # returns releasable pages, preventing unbounded RSS growth over days/weeks. + try: + from hermes_cli.mem_trim import trim_memory + + trim_memory(reason="idle reaper periodic trim") + except Exception as exc: + logger.warning( + "idle reaper memory trim failed: %s: %s", type(exc).__name__, exc + ) def _reclaim_orphaned_leases() -> None: @@ -9826,6 +9839,23 @@ def _interim_assistant_cb(text: str, *, already_streamed: bool = False) -> None: ) _emit("error", sid, {"message": str(e)}) finally: + # Drop both local snapshots of the pre-turn history before asking + # glibc to return pages. session["history"] already points at the + # new/pruned result; retaining either list defeats this trim. + history.clear() + local_run_kwargs = locals().get("run_kwargs") + if isinstance(local_run_kwargs, dict): + local_run_kwargs.clear() + + # Run while any profile-specific HERMES_HOME override is still active + # so context.memory_trim is resolved from the session's own config. + try: + from hermes_cli.mem_trim import trim_memory + + trim_memory(reason="tui turn completion") + except Exception: + logger.debug("post-turn memory trim failed", exc_info=True) + if thinking_started: # Kill the ambient thinking sound the moment the turn ends — # error and success paths both land here. diff --git a/tui_gateway/slash_worker.py b/tui_gateway/slash_worker.py index 5b75dbd16565..85bbb5e5980f 100644 --- a/tui_gateway/slash_worker.py +++ b/tui_gateway/slash_worker.py @@ -20,6 +20,7 @@ import contextlib import io import json +import logging import os import sys import threading @@ -48,6 +49,7 @@ def _env_float(name: str, default: float) -> float: _WATCHDOG_POLL_S = max(0.05, _env_float("HERMES_SLASH_WATCHDOG_POLL_S", 2.0)) _ORPHAN_GRACE_S = max(0.0, _env_float("HERMES_SLASH_WATCHDOG_GRACE_S", 5.0)) _in_flight = threading.Event() # set while a command is executing +logger = logging.getLogger(__name__) def _is_orphaned(original_ppid, getppid=os.getppid) -> bool: @@ -173,6 +175,19 @@ def _sw_log(reason: str) -> None: sys.stdout.flush() finally: _in_flight.clear() + # Workers persist for the TUI session, so release allocator pages at + # the same command boundary as other long-lived gateway processes. + # trim_memory's shared cooldown coalesces this with nearby activity. + try: + from hermes_cli.mem_trim import trim_memory + + trim_memory(reason="slash worker command completion") + except Exception as exc: + logger.warning( + "slash worker memory trim failed: %s: %s", + type(exc).__name__, + exc, + ) if __name__ == "__main__": From 3fad8fdc45b9150173829464b8f075c3cb5171a0 Mon Sep 17 00:00:00 2001 From: kshitij <82637225+kshitijk4poor@users.noreply.github.com> Date: Sun, 2 Aug 2026 22:20:25 +0530 Subject: [PATCH 51/69] polish(mem): readonly config read, debug-level trim logs, loud frame test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Simplify-pass follow-up on the #66355 salvage: 1. _config_settings runs on EVERY trim attempt (before the cooldown check) and only reads — swap load_config for load_config_readonly. Deep-copying the whole config per attempt generates exactly the allocator garbage this module exists to release. Tests re-seamed. 2. Trim-failure logs demoted warning->debug at all 3 periodic sites (gateway housekeeping, idle reaper, slash worker): sibling failure branches in the same loops log at debug, and a persistent failure (e.g. broken import after a partial update) would otherwise warn every 60s forever. 3. The frame-inspection test now asserts the expected locals exist before reading them — a rename in _run_prompt_submit fails the test loudly instead of vacuously passing on None. --- gateway/run.py | 6 +++++- hermes_cli/mem_trim.py | 11 ++++++++--- tests/hermes_cli/test_mem_trim.py | 4 ++-- tests/test_tui_gateway_server.py | 8 +++++++- tui_gateway/server.py | 4 +++- tui_gateway/slash_worker.py | 4 +++- 6 files changed, 28 insertions(+), 9 deletions(-) diff --git a/gateway/run.py b/gateway/run.py index 1a92da6bebfe..e184be76fb99 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -25436,7 +25436,11 @@ def _start_gateway_housekeeping(stop_event: threading.Event, adapters=None, loop trim_memory(reason="messaging gateway housekeeping") except Exception as exc: - logger.warning( + # debug, not warning: sibling housekeeping branches all log + # failures at debug, and a persistent failure (e.g. broken + # import after a partial update) would otherwise warn every + # 60s forever. + logger.debug( "gateway housekeeping memory trim failed: %s: %s", type(exc).__name__, exc, diff --git a/hermes_cli/mem_trim.py b/hermes_cli/mem_trim.py index ff15bea2ecb0..34d2ce364ef0 100644 --- a/hermes_cli/mem_trim.py +++ b/hermes_cli/mem_trim.py @@ -37,9 +37,14 @@ def _config_settings() -> tuple[bool, float, int, float]: log_every_n: Any = _DEFAULT_LOG_EVERY_N info_log_min_delta_mb: Any = _DEFAULT_INFO_LOG_MIN_DELTA_MB try: - from hermes_cli.config import load_config - - config = load_config() or {} + # Read-only access: settings are only .get()ed and coerced, never + # mutated — use the no-deepcopy variant. This runs on EVERY trim + # attempt (before the cooldown check), and generating a full-config + # deepcopy per attempt is exactly the allocator garbage this module + # exists to release. + from hermes_cli.config import load_config_readonly + + config = load_config_readonly() or {} context = config.get("context") if isinstance(config, dict) else None settings = context.get("memory_trim") if isinstance(context, dict) else None if isinstance(settings, dict): diff --git a/tests/hermes_cli/test_mem_trim.py b/tests/hermes_cli/test_mem_trim.py index bf6936022fe4..dd5b38eb3c23 100644 --- a/tests/hermes_cli/test_mem_trim.py +++ b/tests/hermes_cli/test_mem_trim.py @@ -144,7 +144,7 @@ def test_config_cooldown_controls_rate_limit(monkeypatch): monkeypatch.setattr(mem_trim, "_last_trim_monotonic", 1.0) monkeypatch.setattr(mem_trim.time, "monotonic", lambda: 100.0) monkeypatch.setattr( - "hermes_cli.config.load_config", + "hermes_cli.config.load_config_readonly", lambda: { "context": { "memory_trim": {"enabled": True, "cooldown_seconds": 120.0} @@ -161,7 +161,7 @@ def test_legacy_environment_switch_does_not_control_behavior(monkeypatch): monkeypatch.setattr(mem_trim, "_malloc_trim", trim) monkeypatch.setenv("HERMES_DISABLE_MEMORY_TRIM", "1") monkeypatch.setattr( - "hermes_cli.config.load_config", + "hermes_cli.config.load_config_readonly", lambda: {"context": {"memory_trim": {"enabled": True}}}, ) diff --git a/tests/test_tui_gateway_server.py b/tests/test_tui_gateway_server.py index b2d8e918fa1f..c5879a042793 100644 --- a/tests/test_tui_gateway_server.py +++ b/tests/test_tui_gateway_server.py @@ -14186,7 +14186,7 @@ def test_reap_idle_sessions_logs_trim_failure(monkeypatch, caplog): monkeypatch.setattr(mem_trim, "trim_memory", lambda **_kw: (_ for _ in ()).throw(RuntimeError("boom"))) server._sessions.clear() try: - with caplog.at_level("WARNING", logger="tui_gateway.server"): + with caplog.at_level("DEBUG", logger="tui_gateway.server"): server._reap_idle_sessions() assert "idle reaper memory trim failed: RuntimeError: boom" in caplog.text finally: @@ -15610,6 +15610,12 @@ def _inspect_trim_frame(**_kwargs): frame = inspect.currentframe() assert frame is not None and frame.f_back is not None caller_locals = frame.f_back.f_locals + # Loud, not vacuous: if the production locals are ever renamed, fail + # the test instead of silently reading None and "passing". + assert "history" in caller_locals and "run_kwargs" in caller_locals, ( + "expected locals not found in _run_prompt_submit's finally frame — " + "renamed? update this test" + ) observed["history"] = caller_locals.get("history") observed["run_kwargs"] = caller_locals.get("run_kwargs") diff --git a/tui_gateway/server.py b/tui_gateway/server.py index f2e8fb647212..015ec90df3c2 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -1064,7 +1064,9 @@ def _reap_idle_sessions() -> None: trim_memory(reason="idle reaper periodic trim") except Exception as exc: - logger.warning( + # debug, not warning — persistent failure would repeat every reaper + # scan (300s) forever; sibling failure branches log at debug. + logger.debug( "idle reaper memory trim failed: %s: %s", type(exc).__name__, exc ) diff --git a/tui_gateway/slash_worker.py b/tui_gateway/slash_worker.py index 85bbb5e5980f..7066a62f434d 100644 --- a/tui_gateway/slash_worker.py +++ b/tui_gateway/slash_worker.py @@ -183,7 +183,9 @@ def _sw_log(reason: str) -> None: trim_memory(reason="slash worker command completion") except Exception as exc: - logger.warning( + # debug, not warning — a persistent failure would repeat on + # every slash command forever. + logger.debug( "slash worker memory trim failed: %s: %s", type(exc).__name__, exc, From c575351d9ac50f8e976a6c910e07eedd49e09e32 Mon Sep 17 00:00:00 2001 From: kshitij <82637225+kshitijk4poor@users.noreply.github.com> Date: Sun, 2 Aug 2026 22:41:02 +0530 Subject: [PATCH 52/69] fix(mem): 5s force floor so delegate-batch closes don't stack trims MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Efficiency-pass follow-up on the #66355 salvage: force=True bypassed the cooldown entirely, and AIAgent.close() fires a forced trim for EVERY in-process child subagent close (delegate_tool child.close(), parent close step 5). A delegate batch of N children closing back-to-back in the gateway process stacked N+1 uncooled full gc.collect()+malloc_trim passes (50-500ms each with a large live heap). Forced trims now honor a 5s floor — bursts coalesce, the parent's final close-trim still fires. Guard test mutation-checked (floor zeroed -> test fails). --- hermes_cli/mem_trim.py | 12 ++++++++++ tests/hermes_cli/test_mem_trim.py | 38 ++++++++++++++++++++++++++++++- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/hermes_cli/mem_trim.py b/hermes_cli/mem_trim.py index 34d2ce364ef0..ad54f59a38e5 100644 --- a/hermes_cli/mem_trim.py +++ b/hermes_cli/mem_trim.py @@ -200,6 +200,18 @@ def trim_memory( ) if not force and _last_trim_monotonic and now - _last_trim_monotonic < cooldown: return False + # Even forced trims honor a short floor: AIAgent.close() forces a trim, + # and delegate batches close N child subagents back-to-back in the SAME + # process — without a floor that stacks N+1 uncooled full gc.collect() + # passes (50-500ms each in a large gateway process). 5s coalesces the + # burst while keeping the parent's final close-trim effective. + _FORCE_FLOOR_SECONDS = 5.0 + if ( + force + and _last_trim_monotonic + and now - _last_trim_monotonic < _FORCE_FLOOR_SECONDS + ): + return False # Record the attempt before calling into libc so repeated failures do not # turn every turn boundary into an expensive full collection. _last_trim_monotonic = now diff --git a/tests/hermes_cli/test_mem_trim.py b/tests/hermes_cli/test_mem_trim.py index dd5b38eb3c23..3c94f23f07eb 100644 --- a/tests/hermes_cli/test_mem_trim.py +++ b/tests/hermes_cli/test_mem_trim.py @@ -108,7 +108,10 @@ def test_force_logs_even_when_periodic_log_sampling_skips(monkeypatch, caplog): monkeypatch.setattr(mem_trim.gc, "collect", lambda: None) monkeypatch.setattr(mem_trim, "_malloc_trim", lambda _pad: 1) monkeypatch.setattr(mem_trim, "_config_settings", lambda: (True, 0.0, 99, 1.0)) - monkeypatch.setattr(mem_trim.time, "monotonic", lambda: 100.0) + # Two ticks: the forced call comes after the 5s force floor so it runs + # (the floor exists to coalesce burst closes, not to mute logging). + _ticks = iter([100.0, 110.0]) + monkeypatch.setattr(mem_trim.time, "monotonic", lambda: next(_ticks, 110.0)) monkeypatch.setattr( mem_trim, "collect_memory_snapshot", @@ -178,3 +181,36 @@ def test_libc_failure_is_fail_open_and_rate_limited(monkeypatch): assert mem_trim._last_trim_monotonic == 100.0 assert mem_trim.trim_memory(cooldown_seconds=60) is False assert trim.call_count == 1 + + +def test_force_floor_coalesces_burst_closes(monkeypatch): + """A delegate batch closes N child agents back-to-back, each forcing a + trim — the short force floor must coalesce the burst instead of stacking + N uncooled full gc.collect() passes in the same process.""" + collect = Mock() + trim = Mock(return_value=1) + monkeypatch.setattr(mem_trim.gc, "collect", collect) + monkeypatch.setattr(mem_trim, "_malloc_trim", trim) + monkeypatch.setattr(mem_trim, "_config_settings", lambda: (True, 0.0, 1, 0.0)) + monkeypatch.setattr( + mem_trim, + "collect_memory_snapshot", + lambda: {"rss_kib": 4096, "rss_anon_kib": 3072, "thread_count": 3}, + ) + monkeypatch.setattr(mem_trim, "_last_trim_monotonic", 0.0) + + # t=100: first forced close runs. + monkeypatch.setattr(mem_trim.time, "monotonic", lambda: 100.0) + assert mem_trim.trim_memory(force=True, reason="agent close") is True + assert trim.call_count == 1 + + # t=101..103: three more child closes inside the floor — all coalesced. + for t in (101.0, 102.0, 103.0): + monkeypatch.setattr(mem_trim.time, "monotonic", lambda t=t: t) + assert mem_trim.trim_memory(force=True, reason="agent close") is False + assert trim.call_count == 1, "burst closes must not stack forced trims" + + # t=106: past the floor — the parent's final close-trim still fires. + monkeypatch.setattr(mem_trim.time, "monotonic", lambda: 106.0) + assert mem_trim.trim_memory(force=True, reason="agent close") is True + assert trim.call_count == 2 From acfb40c9c773ecc123eeeab4ef2589bdb3d064a8 Mon Sep 17 00:00:00 2001 From: Kent Date: Wed, 13 May 2026 22:09:04 -0500 Subject: [PATCH 53/69] perf(tools): negative-result cache for read_file + search misses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When read_file or search hits a non-existent path, ShellFileOperations spawns a subprocess to stat the path and another to walk the parent directory for "did you mean..." suggestions. A typo'd path retried 13 times (observed in the wild) costs 26 subprocess invocations + 13 ls walks for a result we already know. Add a per-task negative-result cache keyed by (op, resolved_path) with a 60s TTL and a hard cap of 500 entries. On hit, return the cached error JSON immediately and skip the subprocess + suggestion walk. The cache is namespaced by operation ("read" vs "search") because the two callers return different error JSON shapes ("File not found:" vs "Path not found:"). Eviction: * TTL (60s) — short, so a path that appears later isn't masked. * write_file / patch on the same path — _invalidate_dedup_for_path now also drops the negative-cache entry so a freshly-written file is read from disk on the next call instead of returning a stale "not found" stub. Tests in tests/tools/test_file_tools.py cover: * read cache hit skips the subprocess on retry * cache is per-task (no cross-task pollution) * successful reads do not poison the cache * search cache hit skips the subprocess on retry * read and search caches are namespaced (different error shapes) * write_file invalidates the read negative cache * TTL expiry evicts stale entries --- tests/tools/test_file_tools.py | 204 +++++++++++++++++++++++++++++++++ tools/file_tools.py | 112 +++++++++++++++++- 2 files changed, 310 insertions(+), 6 deletions(-) diff --git a/tests/tools/test_file_tools.py b/tests/tools/test_file_tools.py index 329247ec313d..10961949688a 100644 --- a/tests/tools/test_file_tools.py +++ b/tests/tools/test_file_tools.py @@ -716,3 +716,207 @@ def test_invalidate_evicts_the_task_resolved_key(self, tmp_path, monkeypatch): assert correct not in remaining, remaining ft._read_tracker.pop(task_id, None) + + +# --------------------------------------------------------------------------- +# Negative-result cache tests +# +# Without this cache, a typo'd path retried 13 times (observed in the wild) +# spawned 13 wc -c subprocesses + 13 ls walks for the "did you mean..." hint. +# The cache returns the same error JSON immediately and skips both shells. +# --------------------------------------------------------------------------- + +class TestNotFoundCache: + @patch("tools.file_tools._get_file_ops") + def test_read_caches_file_not_found_and_skips_subprocess_on_retry(self, mock_get): + mock_ops = MagicMock() + result_obj = MagicMock() + result_obj.content = None + # Shape returned by ShellFileOperations._suggest_similar_files + result_obj.to_dict.return_value = { + "error": "File not found: /tmp/does-not-exist-neg-1.txt", + "similar_files": [], + } + mock_ops.read_file.return_value = result_obj + mock_get.return_value = mock_ops + + from tools.file_tools import read_file_tool, _read_tracker + # Use a unique task_id so we don't collide with other tests. + tid = "neg-cache-read-1" + _read_tracker.pop(tid, None) + + # First call: subprocess runs, error returned, cache populated. + first = json.loads(read_file_tool("/tmp/does-not-exist-neg-1.txt", task_id=tid)) + assert "File not found" in first["error"] + assert mock_ops.read_file.call_count == 1 + + # Second call: same path → cache hit → no new subprocess call. + second = json.loads(read_file_tool("/tmp/does-not-exist-neg-1.txt", task_id=tid)) + assert "File not found" in second["error"] + assert mock_ops.read_file.call_count == 1, ( + "Negative cache hit must skip the subprocess on retry" + ) + + @patch("tools.file_tools._get_file_ops") + def test_read_cache_isolated_per_task(self, mock_get): + mock_ops = MagicMock() + result_obj = MagicMock() + result_obj.to_dict.return_value = { + "error": "File not found: /tmp/does-not-exist-neg-2.txt", + "similar_files": [], + } + mock_ops.read_file.return_value = result_obj + mock_get.return_value = mock_ops + + from tools.file_tools import read_file_tool, _read_tracker + for tid in ("neg-cache-iso-A", "neg-cache-iso-B"): + _read_tracker.pop(tid, None) + + read_file_tool("/tmp/does-not-exist-neg-2.txt", task_id="neg-cache-iso-A") + read_file_tool("/tmp/does-not-exist-neg-2.txt", task_id="neg-cache-iso-B") + # Each task gets its own miss; B doesn't reuse A's cache entry. + assert mock_ops.read_file.call_count == 2 + + @patch("tools.file_tools._get_file_ops") + def test_read_cache_populated_only_for_not_found(self, mock_get): + # A successful read must NOT populate the negative cache. + mock_ops = MagicMock() + result_obj = MagicMock() + result_obj.content = "x" + result_obj.to_dict.return_value = {"content": "x", "total_lines": 1} + mock_ops.read_file.return_value = result_obj + mock_get.return_value = mock_ops + + from tools.file_tools import read_file_tool, _read_tracker + tid = "neg-cache-success-only" + _read_tracker.pop(tid, None) + + read_file_tool("/tmp/exists-or-mocked.txt", task_id=tid) + nf = _read_tracker[tid].get("not_found", {}) + assert all(k[0] != "read" or "exists-or-mocked" not in k[1] for k in nf), ( + "Successful reads must not poison the negative cache" + ) + + @patch("tools.file_tools._get_file_ops") + def test_search_caches_path_not_found_and_skips_subprocess_on_retry(self, mock_get): + mock_ops = MagicMock() + result_obj = MagicMock() + result_obj.matches = [] + result_obj.to_dict.return_value = { + "error": "Path not found: /tmp/does-not-exist-search-3", + "total_count": 0, + } + mock_ops.search.return_value = result_obj + mock_get.return_value = mock_ops + + from tools.file_tools import search_tool, _read_tracker + tid = "neg-cache-search-3" + _read_tracker.pop(tid, None) + + first = json.loads(search_tool("foo", path="/tmp/does-not-exist-search-3", task_id=tid)) + assert "Path not found" in first["error"] + assert mock_ops.search.call_count == 1 + + second = json.loads(search_tool("foo", path="/tmp/does-not-exist-search-3", task_id=tid)) + assert "Path not found" in second["error"] + assert mock_ops.search.call_count == 1, ( + "Search negative cache hit must skip the subprocess on retry" + ) + + @patch("tools.file_tools._get_file_ops") + def test_read_and_search_caches_are_namespaced(self, mock_get): + # A read that misses must NOT serve a subsequent search call's miss + # (different error JSON shapes). + mock_ops = MagicMock() + + read_obj = MagicMock() + read_obj.to_dict.return_value = { + "error": "File not found: /tmp/does-not-exist-namespace-4", + } + mock_ops.read_file.return_value = read_obj + + search_obj = MagicMock() + search_obj.matches = [] + search_obj.to_dict.return_value = { + "error": "Path not found: /tmp/does-not-exist-namespace-4", + "total_count": 0, + } + mock_ops.search.return_value = search_obj + + mock_get.return_value = mock_ops + + from tools.file_tools import read_file_tool, search_tool, _read_tracker + tid = "neg-cache-namespace-4" + _read_tracker.pop(tid, None) + + read_file_tool("/tmp/does-not-exist-namespace-4", task_id=tid) + search_tool("foo", path="/tmp/does-not-exist-namespace-4", task_id=tid) + # Both ops must hit their own caller (namespacing prevents read's + # error JSON from being returned to search). + assert mock_ops.read_file.call_count == 1 + assert mock_ops.search.call_count == 1 + + @patch("tools.file_tools._get_file_ops") + def test_write_invalidates_read_negative_cache(self, mock_get): + # After write_file on a path, a subsequent read must hit disk, + # not return the cached "not found" stub. + mock_ops = MagicMock() + + not_found_obj = MagicMock() + not_found_obj.to_dict.return_value = { + "error": "File not found: /tmp/will-be-created-neg-5.txt", + } + present_obj = MagicMock() + present_obj.content = "after write" + present_obj.to_dict.return_value = {"content": "after write", "total_lines": 1} + + # First read → not found; second read (after write) → present. + mock_ops.read_file.side_effect = [not_found_obj, present_obj] + write_result_obj = MagicMock() + write_result_obj.to_dict.return_value = {"status": "ok"} + mock_ops.write_file.return_value = write_result_obj + mock_get.return_value = mock_ops + + from tools.file_tools import read_file_tool, write_file_tool, _read_tracker + tid = "neg-cache-write-invalidate-5" + _read_tracker.pop(tid, None) + + first = json.loads(read_file_tool("/tmp/will-be-created-neg-5.txt", task_id=tid)) + assert "File not found" in first["error"] + + write_file_tool("/tmp/will-be-created-neg-5.txt", "after write", task_id=tid) + + second = json.loads(read_file_tool("/tmp/will-be-created-neg-5.txt", task_id=tid)) + assert second.get("content") == "after write", ( + "write_file must invalidate the negative cache so the next read " + "hits the now-existing file instead of returning a stale stub" + ) + assert mock_ops.read_file.call_count == 2 + + def test_not_found_ttl_expires(self): + # A cache entry older than _NOT_FOUND_TTL_SECONDS must be discarded. + from tools.file_tools import ( + _check_not_found_cache, + _record_not_found, + _read_tracker, + _NOT_FOUND_TTL_SECONDS, + ) + import tools.file_tools as ft + + tid = "neg-cache-ttl-6" + _read_tracker.pop(tid, None) + _record_not_found("read", "/tmp/ttl-test", tid, '{"error":"x"}') + # Fresh entry: cache hit. + assert _check_not_found_cache("read", "/tmp/ttl-test", tid) is not None + + # Backdate the entry past the TTL. + with ft._read_tracker_lock: + entry = _read_tracker[tid]["not_found"][("read", "/tmp/ttl-test")] + ft._read_tracker[tid]["not_found"][("read", "/tmp/ttl-test")] = ( + entry[0] - _NOT_FOUND_TTL_SECONDS - 1.0, + entry[1], + ) + # Stale entry: cache miss, also evicted. + assert _check_not_found_cache("read", "/tmp/ttl-test", tid) is None + with ft._read_tracker_lock: + assert ("read", "/tmp/ttl-test") not in _read_tracker[tid].get("not_found", {}) diff --git a/tools/file_tools.py b/tools/file_tools.py index 9a9590c9f7f5..88d79c0c3372 100644 --- a/tools/file_tools.py +++ b/tools/file_tools.py @@ -872,6 +872,8 @@ def _reset_patch_failures(task_id: str, resolved_paths: list) -> None: _READ_HISTORY_CAP = 500 # set; used only by get_read_files_summary _DEDUP_CAP = 1000 # dict; skip-identical-reread guard _READ_TIMESTAMPS_CAP = 1000 # dict; external-edit detection for write/patch +_NOT_FOUND_CAP = 500 # dict; per-task negative-result cache for missing paths +_NOT_FOUND_TTL_SECONDS = 60.0 # short TTL — a path that didn't exist may be created soon _READ_DEDUP_STATUS_MESSAGE = ( "File unchanged since last read. The content from " "the earlier read_file result in this conversation is " @@ -929,6 +931,60 @@ def _cap_read_tracker_data(task_data: dict) -> None: except (StopIteration, KeyError): break + nf = task_data.get("not_found") + if nf is not None and len(nf) > _NOT_FOUND_CAP: + excess = len(nf) - _NOT_FOUND_CAP + for _ in range(excess): + try: + nf.pop(next(iter(nf))) + except (StopIteration, KeyError): + break + + +def _check_not_found_cache(op: str, resolved_str: str, task_id: str) -> str | None: + """Return cached not-found JSON for *(op, resolved_str)* if still fresh. + + Skips the expensive subprocess + suggestion walk when the model retries + the same missing path. Observed in agent.log: a single typo'd path was + retried 13 times — each retry forked a shell to walk the parent directory + and score similar names. + + *op* is "read" or "search" — kept separate because the two callers return + different error JSON shapes ("File not found:" vs "Path not found:"). + + Eviction: TTL or write_file/patch on the path (see invalidate_for_path). + """ + import time + with _read_tracker_lock: + task_data = _read_tracker.get(task_id) + if not task_data: + return None + nf = task_data.get("not_found") + if not nf: + return None + entry = nf.get((op, resolved_str)) + if entry is None: + return None + ts, cached_json = entry + if time.monotonic() - ts > _NOT_FOUND_TTL_SECONDS: + nf.pop((op, resolved_str), None) + return None + return cached_json + + +def _record_not_found(op: str, resolved_str: str, task_id: str, error_json: str) -> None: + """Cache a not-found error so the next *op* call for *resolved_str* skips I/O.""" + import time + with _read_tracker_lock: + task_data = _read_tracker.setdefault(task_id, { + "last_key": None, "consecutive": 0, + "read_history": set(), "dedup": {}, + "dedup_hits": {}, "read_timestamps": {}, + }) + nf = task_data.setdefault("not_found", {}) + nf[(op, resolved_str)] = (time.monotonic(), error_json) + _cap_read_tracker_data(task_data) + def _is_internal_file_status_text(content: str) -> bool: """Return True when content looks like an internal file-tool status, not real file bytes. @@ -1282,6 +1338,15 @@ def read_file_tool(path: str, offset: int = 1, limit: int = 500, task_id: str = if block_error: return tool_error(block_error) + # ── Negative-result cache ───────────────────────────────────── + # If we already discovered this path doesn't exist (within TTL), + # return the cached error without spawning the subprocess + + # similar-files walk. Cleared by write_file/patch on the same path. + resolved_str_for_neg = str(_resolve_path_for_task(path, task_id)) + cached_not_found = _check_not_found_cache("read", resolved_str_for_neg, task_id) + if cached_not_found is not None: + return cached_not_found + # ── Dedup check ─────────────────────────────────────────────── # If we already read this exact (path, offset, limit) and the # file hasn't been modified since, return a lightweight stub @@ -1345,6 +1410,15 @@ def read_file_tool(path: str, offset: int = 1, limit: int = 500, task_id: str = result = file_ops.read_file(path, offset, limit) result_dict = result.to_dict() + # ── Populate negative-result cache on not-found ─────────────── + # _suggest_similar_files returns ReadResult(error="File not found: .."). + # Cache the JSON we'd return so a retry skips the parent-dir walk. + _err = result_dict.get("error") or "" + if isinstance(_err, str) and _err.startswith("File not found:"): + _not_found_json = json.dumps(result_dict, ensure_ascii=False) + _record_not_found("read", resolved_str_for_neg, task_id, _not_found_json) + return _not_found_json + # ── Character-count guard ───────────────────────────────────── # We're model-agnostic so we can't count tokens; characters are # the best proxy we have. If the read produced an unreasonable @@ -1544,12 +1618,18 @@ def _invalidate_dedup_for_path(filepath: str, task_id: str) -> None: if task_data is None: return dedup = task_data.get("dedup") - if not dedup: - return - # Collect keys to remove (can't mutate dict during iteration). - stale_keys = [k for k in dedup if k[0] == resolved] - for k in stale_keys: - del dedup[k] + if dedup: + # Collect keys to remove (can't mutate dict during iteration). + stale_keys = [k for k in dedup if k[0] == resolved] + for k in stale_keys: + del dedup[k] + # Also evict from the negative-result cache: a write_file that + # creates the path means subsequent reads (or searches under it) + # must hit disk. + nf = task_data.get("not_found") + if nf: + nf.pop(("read", resolved), None) + nf.pop(("search", resolved), None) def _update_read_timestamp(filepath: str, task_id: str) -> None: @@ -1974,6 +2054,19 @@ def search_tool(pattern: str, target: str = "content", path: str = ".", if block_error: return tool_error(block_error) + # ── Negative-result cache ───────────────────────────────────── + # Search returns "Path not found: " when the search root + # doesn't exist. The error path also lists the parent directory + # (file_operations.py:1402) — expensive to repeat. Cache so the + # next call to a known-missing root skips both shells. + try: + resolved_search_path = str(_resolve_path_for_task(path, task_id)) + except (OSError, ValueError): + resolved_search_path = path + cached_search_nf = _check_not_found_cache("search", resolved_search_path, task_id) + if cached_search_nf is not None: + return cached_search_nf + file_ops = _get_file_ops(task_id) result = file_ops.search( pattern=pattern, path=path, target=target, file_glob=file_glob, @@ -1992,6 +2085,13 @@ def search_tool(pattern: str, target: str = "content", path: str = ".", "token, cache, or secret-bearing environment files." ) + # Populate negative cache when search root was missing. + _search_err = result_dict.get("error") or "" + if isinstance(_search_err, str) and _search_err.startswith("Path not found:"): + _search_nf_json = json.dumps(result_dict, ensure_ascii=False) + _record_not_found("search", resolved_search_path, task_id, _search_nf_json) + return _search_nf_json + if count >= 3: result_dict["_warning"] = ( f"You have run this exact search {count} times consecutively. " From 4fa8d7bb67dbc7c48332ac042e133ee27c5cdf07 Mon Sep 17 00:00:00 2001 From: kshitij <82637225+kshitijk4poor@users.noreply.github.com> Date: Sun, 2 Aug 2026 22:31:18 +0530 Subject: [PATCH 54/69] fix(tools): staleness + tracking-parity fixes for the not-found cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-ups on the #25387 salvage: 1. CRITICAL: a cached miss survived out-of-band file creation (terminal command, external process) for the full 60s TTL — breaking the common agent pattern 'check for file -> create it -> read it' (live-repro'd). Serve-side existence guard: one ~free stat before serving a cached miss; if the path now exists the entry is evicted and the real read runs. Also fixes the search-root variant (write under a cached-missing directory). Both mutation-checked. 2. notify_other_tool_call now clears the task's not_found entries too (belt: the dispatcher calls it for every non-read tool). 3. Tracking parity: the record sites no longer early-return. On upstream, error results flow through consecutive-loop detection and dedup bookkeeping; short-circuiting skipped that and broke TestDedupInvalidationTaskResolution when preceded by TestSilentFileMisplacementE2E (bisected: the early return at the read record site was the trigger). Recording is now side-effect-identical to upstream; serving from the cache remains the optimization. Also reuse the already-computed _resolved instead of resolving a second time. --- tests/tools/test_file_tools.py | 74 ++++++++++++++++++++++++++++++++++ tools/file_tools.py | 33 +++++++++++++-- 2 files changed, 103 insertions(+), 4 deletions(-) diff --git a/tests/tools/test_file_tools.py b/tests/tools/test_file_tools.py index 10961949688a..1efe65669808 100644 --- a/tests/tools/test_file_tools.py +++ b/tests/tools/test_file_tools.py @@ -920,3 +920,77 @@ def test_not_found_ttl_expires(self): assert _check_not_found_cache("read", "/tmp/ttl-test", tid) is None with ft._read_tracker_lock: assert ("read", "/tmp/ttl-test") not in _read_tracker[tid].get("not_found", {}) + + def test_out_of_band_creation_defeats_cached_miss(self, tmp_path): + """CRITICAL staleness contract: a file created AFTER a cached miss — + by a terminal command or any external process, NOT write_file_tool — + must be served for real on the next read. The agent pattern + 'check for file → create it → read it' breaks otherwise.""" + from tools.file_tools import ( + _check_not_found_cache, + _record_not_found, + _read_tracker, + ) + + tid = "neg-cache-oob-read" + _read_tracker.pop(tid, None) + target = tmp_path / "created-later.txt" + + _record_not_found("read", str(target), tid, '{"error":"File not found: x"}') + assert _check_not_found_cache("read", str(target), tid) is not None + + # Out-of-band creation: plain filesystem write, no tool hook fires. + target.write_text("real content\n") + + # The cached miss must NOT be served once the path exists… + assert _check_not_found_cache("read", str(target), tid) is None, ( + "stale 'File not found' served after the file was created " + "out-of-band — the existence guard regressed" + ) + # …and the entry is evicted, not just skipped. + with __import__("tools.file_tools", fromlist=["x"])._read_tracker_lock: + assert ("read", str(target)) not in _read_tracker[tid].get("not_found", {}) + + def test_out_of_band_creation_defeats_cached_search_miss(self, tmp_path): + """Same contract for search roots: creating a file under a + previously-missing directory must defeat the cached 'Path not found'.""" + from tools.file_tools import ( + _check_not_found_cache, + _record_not_found, + _read_tracker, + ) + + tid = "neg-cache-oob-search" + _read_tracker.pop(tid, None) + missing_dir = tmp_path / "later-dir" + + _record_not_found("search", str(missing_dir), tid, '{"error":"Path not found: x"}') + assert _check_not_found_cache("search", str(missing_dir), tid) is not None + + missing_dir.mkdir() + (missing_dir / "x.txt").write_text("hi\n") + + assert _check_not_found_cache("search", str(missing_dir), tid) is None, ( + "stale 'Path not found' served after the directory was created" + ) + + def test_notify_other_tool_call_clears_not_found(self): + """Belt-and-suspenders: any non-read tool (terminal etc.) invalidates + the task's negative cache via the dispatcher's notify hook.""" + from tools.file_tools import ( + _check_not_found_cache, + _record_not_found, + _read_tracker, + notify_other_tool_call, + ) + + tid = "neg-cache-notify" + _read_tracker.pop(tid, None) + _record_not_found("read", "/tmp/never-exists-notify", tid, '{"error":"x"}') + assert _check_not_found_cache("read", "/tmp/never-exists-notify", tid) is not None + + notify_other_tool_call(tid) + + assert _check_not_found_cache("read", "/tmp/never-exists-notify", tid) is None, ( + "notify_other_tool_call must clear cached misses" + ) diff --git a/tools/file_tools.py b/tools/file_tools.py index 88d79c0c3372..1078d736dfa2 100644 --- a/tools/file_tools.py +++ b/tools/file_tools.py @@ -954,6 +954,7 @@ def _check_not_found_cache(op: str, resolved_str: str, task_id: str) -> str | No Eviction: TTL or write_file/patch on the path (see invalidate_for_path). """ + import os as _os import time with _read_tracker_lock: task_data = _read_tracker.get(task_id) @@ -969,6 +970,15 @@ def _check_not_found_cache(op: str, resolved_str: str, task_id: str) -> str | No if time.monotonic() - ts > _NOT_FOUND_TTL_SECONDS: nf.pop((op, resolved_str), None) return None + # Existence guard: the path may have been created since we cached + # the miss — by a terminal command, another agent, or any external + # process (write_file/patch invalidate explicitly, but they're not + # the only writers). The agent pattern "check file → create it → + # read it" is common; serving a stale miss for up to the TTL breaks + # it. One stat is ~free next to the subprocess walk we're skipping. + if _os.path.exists(resolved_str): + nf.pop((op, resolved_str), None) + return None return cached_json @@ -1342,7 +1352,7 @@ def read_file_tool(path: str, offset: int = 1, limit: int = 500, task_id: str = # If we already discovered this path doesn't exist (within TTL), # return the cached error without spawning the subprocess + # similar-files walk. Cleared by write_file/patch on the same path. - resolved_str_for_neg = str(_resolve_path_for_task(path, task_id)) + resolved_str_for_neg = str(_resolved) cached_not_found = _check_not_found_cache("read", resolved_str_for_neg, task_id) if cached_not_found is not None: return cached_not_found @@ -1413,11 +1423,16 @@ def read_file_tool(path: str, offset: int = 1, limit: int = 500, task_id: str = # ── Populate negative-result cache on not-found ─────────────── # _suggest_similar_files returns ReadResult(error="File not found: .."). # Cache the JSON we'd return so a retry skips the parent-dir walk. + # Deliberately NO early return: on upstream, error results flow + # through the tracking block below (consecutive-loop detection, + # dedup bookkeeping via the resolved path) and the normal exit — + # short-circuiting here changes that behavior (and broke a real + # test interaction). Serving from the cache (above) is the + # optimization; recording must stay side-effect-identical. _err = result_dict.get("error") or "" if isinstance(_err, str) and _err.startswith("File not found:"): _not_found_json = json.dumps(result_dict, ensure_ascii=False) _record_not_found("read", resolved_str_for_neg, task_id, _not_found_json) - return _not_found_json # ── Character-count guard ───────────────────────────────────── # We're model-agnostic so we can't count tokens; characters are @@ -1594,6 +1609,15 @@ def notify_other_tool_call(task_id: str = "default"): # progress, so clear per-key dedup hit counters too. if "dedup_hits" in task_data: task_data["dedup_hits"].clear() + # Any other tool (terminal, delegate, ...) may have created a + # previously-missing path — a cached miss is no longer + # trustworthy. The serve-side existence guard in + # _check_not_found_cache already covers this, but clearing + # here keeps the cache honest and covers exotic cases the + # stat can't (e.g. permission flips). + nf = task_data.get("not_found") + if nf: + nf.clear() def _invalidate_dedup_for_path(filepath: str, task_id: str) -> None: @@ -2085,12 +2109,13 @@ def search_tool(pattern: str, target: str = "content", path: str = ".", "token, cache, or secret-bearing environment files." ) - # Populate negative cache when search root was missing. + # Populate negative cache when search root was missing. No early + # return — same rationale as the read path: error results keep + # flowing through the consecutive-search bookkeeping below. _search_err = result_dict.get("error") or "" if isinstance(_search_err, str) and _search_err.startswith("Path not found:"): _search_nf_json = json.dumps(result_dict, ensure_ascii=False) _record_not_found("search", resolved_search_path, task_id, _search_nf_json) - return _search_nf_json if count >= 3: result_dict["_warning"] = ( From 9b50a99b395b46ec7cb97ed823f59b717aca18a1 Mon Sep 17 00:00:00 2001 From: kshitij <82637225+kshitijk4poor@users.noreply.github.com> Date: Sun, 2 Aug 2026 22:41:40 +0530 Subject: [PATCH 55/69] refactor(tools): existence stat outside the global tracker lock Simplify-pass advisory on the #25387 salvage: _read_tracker_lock is one global lock guarding every task's read/search bookkeeping (15 sites). A hung stat on a dead network mount inside it would stall all of them. Check-then-recheck matches the dedup mtime pattern 30 lines below: read the entry under the lock, stat outside, reacquire only to evict. --- tools/file_tools.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/tools/file_tools.py b/tools/file_tools.py index 1078d736dfa2..ad3050efd5ab 100644 --- a/tools/file_tools.py +++ b/tools/file_tools.py @@ -970,16 +970,25 @@ def _check_not_found_cache(op: str, resolved_str: str, task_id: str) -> str | No if time.monotonic() - ts > _NOT_FOUND_TTL_SECONDS: nf.pop((op, resolved_str), None) return None - # Existence guard: the path may have been created since we cached - # the miss — by a terminal command, another agent, or any external - # process (write_file/patch invalidate explicitly, but they're not - # the only writers). The agent pattern "check file → create it → - # read it" is common; serving a stale miss for up to the TTL breaks - # it. One stat is ~free next to the subprocess walk we're skipping. - if _os.path.exists(resolved_str): - nf.pop((op, resolved_str), None) - return None - return cached_json + # Existence guard: the path may have been created since we cached the + # miss — by a terminal command, another agent, or any external process + # (write_file/patch invalidate explicitly, but they're not the only + # writers). The agent pattern "check file → create it → read it" is + # common; serving a stale miss for up to the TTL breaks it. One stat is + # ~free next to the subprocess walk we're skipping. + # + # The stat runs OUTSIDE _read_tracker_lock (matching the dedup mtime + # check below in read_file_tool): the lock is global across all tasks, + # and a hung stat on a dead network mount must not stall every other + # task's read/search bookkeeping. + if _os.path.exists(resolved_str): + with _read_tracker_lock: + task_data = _read_tracker.get(task_id) + nf = task_data.get("not_found") if task_data else None + if nf: + nf.pop((op, resolved_str), None) + return None + return cached_json def _record_not_found(op: str, resolved_str: str, task_id: str, error_json: str) -> None: From 013779924fc5e682a2833ae35689be0645d11b19 Mon Sep 17 00:00:00 2001 From: Josh Tsai <128559392+bounce12340@users.noreply.github.com> Date: Thu, 23 Jul 2026 16:35:30 +0800 Subject: [PATCH 56/69] fix(agent): fail fast on custom-provider /models auth errors - Short-circuit the candidate waterfall on HTTP 401/403: an auth wall proves the endpoint family exists, so probing the alternate URL just doubles the wasted wait (the reported endpoint takes ~10s to return 401 without a key). - Stream the probe so 4xx never downloads a slow error body; responses are closed on every exit path. - Regression tests: single-call assertion on 401/403 (fails on main), negative-cache reuse, 404 waterfall preserved, no .json() on 4xx. Fixes #69905 Co-Authored-By: Claude Fable 5 --- agent/model_metadata.py | 19 +++++++- tests/agent/test_model_metadata.py | 73 ++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 1 deletion(-) diff --git a/agent/model_metadata.py b/agent/model_metadata.py index 42d841009fa3..6643ec820a6e 100644 --- a/agent/model_metadata.py +++ b/agent/model_metadata.py @@ -1185,8 +1185,22 @@ def fetch_endpoint_model_metadata( for candidate in candidates: url = candidate.rstrip("/") + "/models" + response = None try: - response = requests.get(url, headers=headers, timeout=(5, 10), verify=_resolve_requests_verify()) + response = requests.get( + url, + headers=headers, + timeout=(5, 10), + verify=_resolve_requests_verify(), + stream=True, + ) + if response.status_code in (401, 403): + logger.debug( + "Model metadata probe received HTTP %s from %s; stopping candidate probing", + response.status_code, + url, + ) + break response.raise_for_status() payload = response.json() cache: Dict[str, Dict[str, Any]] = {} @@ -1236,6 +1250,9 @@ def fetch_endpoint_model_metadata( return cache except Exception as exc: last_error = exc + finally: + if response is not None: + response.close() if last_error: logger.debug("Failed to fetch model metadata from %s/models: %s", normalized, last_error) diff --git a/tests/agent/test_model_metadata.py b/tests/agent/test_model_metadata.py index ac8ed899cc3e..874b97d5dddc 100644 --- a/tests/agent/test_model_metadata.py +++ b/tests/agent/test_model_metadata.py @@ -489,6 +489,79 @@ def test_live_codex_context_replaces_stale_cache_in_both_directions( +# ========================================================================= +# Custom endpoint model metadata +# ========================================================================= + +class TestFetchEndpointModelMetadata: + def setup_method(self): + import agent.model_metadata as mm + mm._endpoint_model_metadata_cache.clear() + mm._endpoint_model_metadata_cache_time.clear() + + @pytest.mark.parametrize("status_code", [401, 403]) + def test_auth_failure_stops_after_first_candidate(self, status_code): + import agent.model_metadata as mm + + response = MagicMock() + response.status_code = status_code + response.raise_for_status.side_effect = RuntimeError(str(status_code)) + + with patch("agent.model_metadata.requests.get", return_value=response) as mock_get: + result = mm.fetch_endpoint_model_metadata("https://custom.example/v1") + + assert result == {} + mock_get.assert_called_once() + assert mock_get.call_args.kwargs["stream"] is True + response.raise_for_status.assert_not_called() + response.json.assert_not_called() + response.close.assert_called_once() + + def test_auth_failure_empty_result_is_cached(self): + import agent.model_metadata as mm + + response = MagicMock() + response.status_code = 401 + response.raise_for_status.side_effect = RuntimeError("401") + + with patch("agent.model_metadata.requests.get", return_value=response) as mock_get: + first = mm.fetch_endpoint_model_metadata("https://custom.example/v1") + second = mm.fetch_endpoint_model_metadata("https://custom.example/v1") + + assert first == second == {} + mock_get.assert_called_once() + response.close.assert_called_once() + + def test_not_found_still_tries_alternate_candidate(self): + import agent.model_metadata as mm + + not_found = MagicMock() + not_found.status_code = 404 + not_found.raise_for_status.side_effect = RuntimeError("404") + success = MagicMock() + success.status_code = 200 + success.json.return_value = { + "data": [{"id": "test/model", "context_length": 32768}] + } + + with patch( + "agent.model_metadata.requests.get", + side_effect=[not_found, success], + ) as mock_get: + result = mm.fetch_endpoint_model_metadata("https://custom.example/v1") + + assert result["test/model"]["context_length"] == 32768 + assert mock_get.call_count == 2 + assert [call.args[0] for call in mock_get.call_args_list] == [ + "https://custom.example/v1/models", + "https://custom.example/models", + ] + assert all(call.kwargs["stream"] is True for call in mock_get.call_args_list) + not_found.json.assert_not_called() + not_found.close.assert_called_once() + success.close.assert_called_once() + + # ========================================================================= # Nous Portal context-window resolution (provider="nous") # ========================================================================= From 9d7c2d450ccc6e08c769a0e4908d50841711429c Mon Sep 17 00:00:00 2001 From: AMATH <116212274+amathxbt@users.noreply.github.com> Date: Sat, 9 May 2026 01:50:11 +0100 Subject: [PATCH 57/69] fix: _check_disk_usage_warning runs full rglob on every terminal call --- tools/terminal_tool.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/tools/terminal_tool.py b/tools/terminal_tool.py index 6572174819af..1768f80aed3c 100644 --- a/tools/terminal_tool.py +++ b/tools/terminal_tool.py @@ -189,8 +189,24 @@ def _check_vercel_sandbox_requirements(config: dict[str, Any]) -> bool: return False +# Cache for disk usage warning to avoid full rglob scan on every call. +# The check is advisory-only — staleness for up to 5 minutes is acceptable. +_disk_usage_cache: dict = {"timestamp": 0.0, "result": False} +_DISK_USAGE_CACHE_TTL = 300.0 # seconds + + def _check_disk_usage_warning(): - """Check if total disk usage exceeds warning threshold.""" + """Check if total disk usage exceeds warning threshold. + + Result is cached for :data:`_DISK_USAGE_CACHE_TTL` seconds (default: + 5 minutes) to avoid an expensive recursive filesystem scan on every + terminal command. The check is advisory-only so a stale result is + harmless. + """ + import time as _time_mod + now = _time_mod.monotonic() + if now - _disk_usage_cache["timestamp"] < _DISK_USAGE_CACHE_TTL: + return _disk_usage_cache["result"] try: scratch_dir = _get_scratch_dir() @@ -207,14 +223,16 @@ def _check_disk_usage_warning(): total_gb = total_bytes / (1024 ** 3) - if total_gb > DISK_USAGE_WARNING_THRESHOLD_GB: + exceeded = total_gb > DISK_USAGE_WARNING_THRESHOLD_GB + if exceeded: logger.warning("Disk usage (%.1fGB) exceeds threshold (%.0fGB). Consider running cleanup_all_environments().", total_gb, DISK_USAGE_WARNING_THRESHOLD_GB) - return True - - return False + _disk_usage_cache["timestamp"] = _time_mod.monotonic() + _disk_usage_cache["result"] = exceeded + return exceeded except Exception as e: logger.debug("Disk usage warning check failed: %s", e, exc_info=True) + # Don't update cache on error so the next call retries. return False From 69f68e5f609aae90d43e8be936fa9c66da0cd10f Mon Sep 17 00:00:00 2001 From: kshitij <82637225+kshitijk4poor@users.noreply.github.com> Date: Sun, 2 Aug 2026 23:03:10 +0530 Subject: [PATCH 58/69] chore: map jesse.casco@gmail.com to FixItFoundry --- contributors/emails/jesse.casco@gmail.com | 1 + 1 file changed, 1 insertion(+) create mode 100644 contributors/emails/jesse.casco@gmail.com diff --git a/contributors/emails/jesse.casco@gmail.com b/contributors/emails/jesse.casco@gmail.com new file mode 100644 index 000000000000..c19672cddfde --- /dev/null +++ b/contributors/emails/jesse.casco@gmail.com @@ -0,0 +1 @@ +FixItFoundry From 47fa4385df76b4f6937c48e3d155223b1086c373 Mon Sep 17 00:00:00 2001 From: lost9999 Date: Sat, 25 Jul 2026 20:28:43 +0800 Subject: [PATCH 59/69] fix(serve): cache /api/status profile-gateway topology scan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /api/status is the desktop's boot liveness probe (polled ~1/s) but since #60537 every call ran a full topology scan — per-profile yaml.safe_load (pure-Python loader), psutil process probes, realpath walks — in the default executor. On multi-profile installs concurrent polls pile up and hold the GIL 14-16s, starving the event loop: the WS sidecar cannot flush gateway.ready, the desktop times out into the next stall, and boot escalates to the 'Hermes couldn't start' overlay (#60800). Memoize the scan behind a 10s TTL with a collapse lock so concurrent polls share one scan. Topology only changes on gateway start/stop, so a <=10s stale badge is an acceptable trade for not starving the loop. The cache also keys on the collector's identity: tests monkeypatch _collect_profile_gateway_topology per case, and the identity check keeps them hermetic (a swapped collector is a miss) without a reset hook. py-spy captures during a failing boot land in _profile_platform_ports -> yaml.safe_load on executor threads (7 profiles, Windows). After: one cold-start scan, zero recurring stalls, desktop boots. --- hermes_cli/web_server.py | 44 ++++++- .../test_web_server_status_topology_cache.py | 117 ++++++++++++++++++ 2 files changed, 160 insertions(+), 1 deletion(-) create mode 100644 tests/test_web_server_status_topology_cache.py diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index 9e7bf8421abe..857f89605ba2 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -2911,6 +2911,48 @@ def _collect_profile_gateway_topology() -> Dict[str, Any]: return {"profiles": profile_names, "gateway_mode": mode, "gateways": gateways} +# /api/status is polled ~1/s by the desktop app while it waits for the backend +# (and again by the dashboard badge). Each uncached call above walks 7+ profile +# homes (yaml.safe_load with the pure-Python loader + psutil process-table +# probes + realpath walks) inside the default executor; concurrent polls pile +# up and hold the GIL for 14-16s, starving the event loop — the desktop WS +# never receives gateway.ready and boot fails ("event loop stalled ... GIL +# pressure suspected"). Topology changes on gateway start/stop, so a short TTL +# cache with a collapse lock keeps the scan to one per window. The cache also +# remembers which collector produced the entry: tests monkeypatch +# _collect_profile_gateway_topology per case, and the identity check keeps +# them hermetic without needing a reset hook (a swapped collector is a miss). +_TOPOLOGY_CACHE: Dict[str, Any] = {"ts": 0.0, "data": None, "fn": None} +_TOPOLOGY_CACHE_LOCK = threading.Lock() +_TOPOLOGY_CACHE_TTL = 10.0 + + +def _topology_cache_get(fn: Any) -> Optional[Dict[str, Any]]: + if ( + _TOPOLOGY_CACHE["data"] is not None + and _TOPOLOGY_CACHE["fn"] is fn + and time.monotonic() - _TOPOLOGY_CACHE["ts"] < _TOPOLOGY_CACHE_TTL + ): + return _TOPOLOGY_CACHE["data"] + return None + + +def _collect_profile_gateway_topology_cached() -> Dict[str, Any]: + fn = _collect_profile_gateway_topology + cached = _topology_cache_get(fn) + if cached is not None: + return cached + with _TOPOLOGY_CACHE_LOCK: + cached = _topology_cache_get(fn) + if cached is not None: + return cached + data = fn() + _TOPOLOGY_CACHE["data"] = data + _TOPOLOGY_CACHE["fn"] = fn + _TOPOLOGY_CACHE["ts"] = time.monotonic() + return data + + @app.get("/api/ssh/ownership") async def get_ssh_ownership(request: Request): _require_token(request) @@ -3237,7 +3279,7 @@ def _bounded_health_probe(): # per-gateway ``gateways[]`` detail carries host ports (deployment # recon), so it stays gated with the host paths / PID below. topology = await asyncio.get_running_loop().run_in_executor( - None, _collect_profile_gateway_topology + None, _collect_profile_gateway_topology_cached ) status["profiles"] = topology["profiles"] status["gateway_mode"] = topology["gateway_mode"] diff --git a/tests/test_web_server_status_topology_cache.py b/tests/test_web_server_status_topology_cache.py new file mode 100644 index 000000000000..76c4d0be2ed0 --- /dev/null +++ b/tests/test_web_server_status_topology_cache.py @@ -0,0 +1,117 @@ +"""Regression tests for the /api/status profile-topology cache. + +The desktop app polls /api/status ~1/s while waiting for the backend to become +ready. Before the cache, every poll ran a full _collect_profile_gateway_topology +scan (per-profile yaml.safe_load with the pure-Python loader + psutil +process-table probes + realpath walks) in the default executor; on multi-profile +installs the concurrent scans held the GIL for 14-16s and starved the event +loop, so the desktop WS never received gateway.ready and boot escalated to the +"Hermes couldn't start" overlay (#60800). +""" + +import threading +import time + +from hermes_cli import web_server + + +def _reset_cache(): + web_server._TOPOLOGY_CACHE["ts"] = 0.0 + web_server._TOPOLOGY_CACHE["data"] = None + web_server._TOPOLOGY_CACHE["fn"] = None + + +def _fake_topology(calls, delay=0.0): + def _collect(): + if delay: + time.sleep(delay) + calls.append(1) + return {"profiles": ["default"], "gateway_mode": "single", "gateways": []} + + return _collect + + +def test_topology_cache_returns_cached_result_within_ttl(monkeypatch): + calls = [] + monkeypatch.setattr( + web_server, "_collect_profile_gateway_topology", _fake_topology(calls) + ) + _reset_cache() + try: + first = web_server._collect_profile_gateway_topology_cached() + second = web_server._collect_profile_gateway_topology_cached() + finally: + _reset_cache() + + assert len(calls) == 1 + assert first is second + + +def test_topology_cache_rescans_after_ttl(monkeypatch): + calls = [] + monkeypatch.setattr( + web_server, "_collect_profile_gateway_topology", _fake_topology(calls) + ) + _reset_cache() + try: + web_server._collect_profile_gateway_topology_cached() + # Age the cache entry past the TTL instead of sleeping through it. + web_server._TOPOLOGY_CACHE["ts"] -= web_server._TOPOLOGY_CACHE_TTL + 1.0 + web_server._collect_profile_gateway_topology_cached() + finally: + _reset_cache() + + assert len(calls) == 2 + + +def test_topology_cache_collapses_concurrent_scans(monkeypatch): + """Concurrent status polls must not each run their own scan — that pile-up + is exactly the GIL storm the cache exists to prevent.""" + calls = [] + monkeypatch.setattr( + web_server, + "_collect_profile_gateway_topology", + _fake_topology(calls, delay=0.05), + ) + _reset_cache() + results = [] + try: + threads = [ + threading.Thread( + target=lambda: results.append( + web_server._collect_profile_gateway_topology_cached() + ) + ) + for _ in range(8) + ] + for t in threads: + t.start() + for t in threads: + t.join() + finally: + _reset_cache() + + assert len(calls) == 1 + assert len(results) == 8 + assert all(r == results[0] for r in results) +def test_topology_cache_misses_when_collector_is_swapped(monkeypatch): + """Tests (and hot-reload scenarios) monkeypatch the collector; a swapped + function identity must be a cache miss so stale data from the previous + collector never leaks across the swap.""" + calls_a, calls_b = [], [] + monkeypatch.setattr( + web_server, "_collect_profile_gateway_topology", _fake_topology(calls_a) + ) + _reset_cache() + try: + first = web_server._collect_profile_gateway_topology_cached() + monkeypatch.setattr( + web_server, "_collect_profile_gateway_topology", _fake_topology(calls_b) + ) + second = web_server._collect_profile_gateway_topology_cached() + finally: + _reset_cache() + + assert len(calls_a) == 1 + assert len(calls_b) == 1 + assert first is not second From af077ef0396726f4daab467bf559e28a5f2abe10 Mon Sep 17 00:00:00 2001 From: Frowtek Date: Thu, 16 Jul 2026 17:51:23 +0300 Subject: [PATCH 60/69] fix(api_server): run the cron-fire token verifier off the event loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _handle_cron_fire verified the NAS-minted fire JWT by calling the fire-verifier inline on the event loop. That verifier resolves the NAS signing key from a JWKS URL — a synchronous HTTP GET on a cache miss (a cold PyJWKClient, or a rotated kid the cached client doesn't know) — so a slow or rate-limited portal stalls the whole event loop and starves every other adapter sharing it. #64641 already documented this exact symptom (relay 504s on high-job-count instances) and cut the fetch frequency by caching the client per URL, but the residual cache-miss fetch still ran inline on the loop. Dispatch the verifier the same way the platform HTTP event verifier was hardened: await a coroutine verifier directly, run a sync one via asyncio.to_thread so its blocking I/O stays off the loop, and fail closed (reject with 401, never admit the fire) if the verifier raises — this is the only inbound that can trigger remote job execution. The verifier's JWK-client cache is already thread-safe (threading.Lock), so moving the call to a worker thread is safe. Adds regression tests: a sync verifier runs on a worker thread rather than the loop thread, a crashing verifier yields 401 with no fire, and a coroutine verifier is awaited. --- gateway/platforms/api_server.py | 19 +++- tests/gateway/test_cron_fire_webhook.py | 115 ++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 1 deletion(-) diff --git a/gateway/platforms/api_server.py b/gateway/platforms/api_server.py index bd69e203f242..d81aab5a9bd2 100644 --- a/gateway/platforms/api_server.py +++ b/gateway/platforms/api_server.py @@ -5615,12 +5615,29 @@ async def _handle_cron_fire(self, request: "web.Request") -> "web.Response": token = auth[7:].strip() if auth.startswith("Bearer ") else "" cfg = load_config() - claims = get_fire_verifier()( + verifier = get_fire_verifier() + verify_kwargs = dict( token=token, expected_audience=cfg_get(cfg, "cron", "chronos", "expected_audience", default=""), jwks_or_key=cfg_get(cfg, "cron", "chronos", "nas_jwks_url", default="") or None, issuer=cfg_get(cfg, "cron", "chronos", "portal_url", default="") or None, ) + try: + if asyncio.iscoroutinefunction(verifier): + claims = await verifier(**verify_kwargs) + else: + # The verifier resolves the NAS signing key from a JWKS URL, + # which is a synchronous HTTP GET on a cache miss (cold client + # or a rotated kid) — keep that blocking I/O off the event loop + # so a slow or rate-limited portal can't stall every other + # adapter sharing this loop. Same hardening the platform HTTP + # event verifier already got. + claims = await asyncio.to_thread(verifier, **verify_kwargs) + except Exception: + # Fail closed: a crashing verifier must never admit a fire — this + # is the only inbound that can trigger remote job execution. + logger.exception("cron fire: verifier crashed; rejecting token") + claims = None if claims is None: logger.warning( "cron fire: rejected invalid token: %s", diff --git a/tests/gateway/test_cron_fire_webhook.py b/tests/gateway/test_cron_fire_webhook.py index fe61cea767f6..b8f8a3d4cf17 100644 --- a/tests/gateway/test_cron_fire_webhook.py +++ b/tests/gateway/test_cron_fire_webhook.py @@ -123,3 +123,118 @@ async def test_missing_job_id_400(adapter, monkeypatch): assert spy.fired == [] +@pytest.mark.asyncio +async def test_fire_does_not_require_api_server_key(adapter, monkeypatch): + """The fire endpoint must NOT gate on API_SERVER_KEY — auth is the NAS-JWT. + A request with NO API key header but a valid fire token still succeeds.""" + spy = _SpyProvider() + monkeypatch.setattr("cron.scheduler_provider.resolve_cron_scheduler", lambda: spy) + monkeypatch.setattr( + "plugins.cron_providers.chronos.verify.get_fire_verifier", + lambda: (lambda **kw: {"purpose": "cron_fire"}), + ) + + app = _create_app(adapter) + async with TestClient(TestServer(app)) as cli: + # Bearer is the FIRE token, not the API_SERVER_KEY "sk-secret". + resp = await cli.post("/api/cron/fire", + headers={"Authorization": "Bearer nas-jwt"}, + json={"job_id": "j9"}) + assert resp.status == 202 + for _ in range(50): + if spy.fired: + break + await asyncio.sleep(0.01) + assert spy.fired == ["j9"] + + +@pytest.mark.asyncio +async def test_sync_verifier_runs_off_the_event_loop(adapter, monkeypatch): + """The verifier resolves the signing key from a JWKS URL — a synchronous + HTTP GET on a cache miss. It must run via asyncio.to_thread, NOT inline on + the event loop, or a slow/rate-limited portal stalls every other adapter + sharing the loop. Proof: the sync verifier executes on a worker thread, not + the loop thread. + """ + loop_thread_id = threading.get_ident() + seen = {} + + def blocking_verifier(**kw): + seen["thread_id"] = threading.get_ident() + return {"purpose": "cron_fire"} + + spy = _SpyProvider() + monkeypatch.setattr("cron.scheduler_provider.resolve_cron_scheduler", lambda: spy) + monkeypatch.setattr( + "plugins.cron_providers.chronos.verify.get_fire_verifier", + lambda: blocking_verifier, + ) + + app = _create_app(adapter) + async with TestClient(TestServer(app)) as cli: + resp = await cli.post("/api/cron/fire", + headers={"Authorization": "Bearer good"}, + json={"job_id": "off-loop"}) + assert resp.status == 202 + + # If the verifier had run inline on the loop, its thread id would equal the + # loop thread's; to_thread puts it on a distinct worker thread. + assert seen["thread_id"] != loop_thread_id + + +@pytest.mark.asyncio +async def test_crashing_verifier_fails_closed_401(adapter, monkeypatch): + """A verifier that raises must be treated as a rejection (401), never admit + the fire, and never surface as a 500 — this is the only inbound that can + trigger remote job execution, so it fails closed. + """ + spy = _SpyProvider() + monkeypatch.setattr("cron.scheduler_provider.resolve_cron_scheduler", lambda: spy) + + def exploding_verifier(**kw): + raise RuntimeError("JWKS endpoint unreachable") + + monkeypatch.setattr( + "plugins.cron_providers.chronos.verify.get_fire_verifier", + lambda: exploding_verifier, + ) + + app = _create_app(adapter) + async with TestClient(TestServer(app)) as cli: + resp = await cli.post("/api/cron/fire", + headers={"Authorization": "Bearer boom"}, + json={"job_id": "abc123"}) + assert resp.status == 401 + + await asyncio.sleep(0.05) + assert spy.fired == [] + + +@pytest.mark.asyncio +async def test_async_verifier_is_awaited(adapter, monkeypatch): + """A coroutine verifier (a future async escape-hatch) is awaited directly + rather than dispatched to a thread — a valid async verify still fires. + """ + spy = _SpyProvider() + monkeypatch.setattr("cron.scheduler_provider.resolve_cron_scheduler", lambda: spy) + + async def async_verifier(**kw): + return {"purpose": "cron_fire", "aud": "agent:x"} + + monkeypatch.setattr( + "plugins.cron_providers.chronos.verify.get_fire_verifier", + lambda: async_verifier, + ) + + app = _create_app(adapter) + async with TestClient(TestServer(app)) as cli: + resp = await cli.post("/api/cron/fire", + headers={"Authorization": "Bearer good"}, + json={"job_id": "async-ok"}) + assert resp.status == 202 + + for _ in range(50): + if spy.fired: + break + await asyncio.sleep(0.01) + assert spy.fired == ["async-ok"] From 94a0a062146a8094ff6b3630f34de1cc3c7fc138 Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Sun, 2 Aug 2026 23:14:35 +0530 Subject: [PATCH 61/69] chore: add contributor email mapping for wayne1992127 --- contributors/emails/wayne1992127@gmail.com | 1 + 1 file changed, 1 insertion(+) create mode 100644 contributors/emails/wayne1992127@gmail.com diff --git a/contributors/emails/wayne1992127@gmail.com b/contributors/emails/wayne1992127@gmail.com new file mode 100644 index 000000000000..140d69963812 --- /dev/null +++ b/contributors/emails/wayne1992127@gmail.com @@ -0,0 +1 @@ +wayne1992127 From 9eb8e20c684730e0853b3fcabe63ce8d88dd04c3 Mon Sep 17 00:00:00 2001 From: AlexFucuson9 Date: Tue, 30 Jun 2026 17:08:34 +0700 Subject: [PATCH 62/69] fix: use lazy logging with %s formatting in logger calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace f-string interpolation in logger calls with lazy %-style formatting across 10 files (38 instances). This follows Python logging best practices — the message is only formatted if the log level is enabled, avoiding unnecessary string concatenation overhead. Files changed: - trajectory_compressor.py (6) - mini_swe_runner.py (2) - agent/tool_executor.py (1) - agent/model_metadata.py (1) - agent/agent_runtime_helpers.py (3) - agent/chat_completion_helpers.py (3) - agent/conversation_loop.py (8) - tools/skills_hub.py (2) - tools/environments/docker.py (10) - gateway/kanban_watchers.py (2) --- agent/agent_runtime_helpers.py | 6 +++--- agent/chat_completion_helpers.py | 6 +++--- agent/conversation_loop.py | 16 ++++++++-------- agent/model_metadata.py | 2 +- gateway/kanban_watchers.py | 4 ++-- mini_swe_runner.py | 4 ++-- tools/environments/docker.py | 20 ++++++++++---------- tools/skills_hub.py | 4 ++-- trajectory_compressor.py | 12 ++++++------ 9 files changed, 37 insertions(+), 37 deletions(-) diff --git a/agent/agent_runtime_helpers.py b/agent/agent_runtime_helpers.py index 12bed0c666fd..579314018cde 100644 --- a/agent/agent_runtime_helpers.py +++ b/agent/agent_runtime_helpers.py @@ -151,7 +151,7 @@ def convert_to_trajectory_format(agent, messages: List[Dict[str, Any]], user_que except json.JSONDecodeError: # This shouldn't happen since we validate and retry during conversation, # but if it does, log warning and use empty dict - logger.warning(f"Unexpected invalid JSON in trajectory conversion: {tool_call['function']['arguments'][:100]}") + logger.warning("Unexpected invalid JSON in trajectory conversion: %s", tool_call['function']['arguments'][:100]) arguments = {} tool_call_json = { @@ -1210,7 +1210,7 @@ def _rotate_failed_credential(rotate_status: int): refreshed_id, ) return False, has_retried_429 - _ra().logger.info(f"Credential auth failure — refreshed pool entry {getattr(refreshed, 'id', '?')}") + _ra().logger.info("Credential auth failure — refreshed pool entry %s", getattr(refreshed, 'id', '?')) agent._swap_credential(refreshed) return True, has_retried_429 # Refresh failed — rotate to next credential instead of giving up. @@ -1835,7 +1835,7 @@ def dump_api_request_debug( return dump_file except Exception as dump_error: if agent.verbose_logging: - logger.warning(f"Failed to dump API request debug payload: {dump_error}") + logger.warning("Failed to dump API request debug payload: %s", dump_error) return None diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.py index e40333f0e601..e6e8ab7fdc1f 100644 --- a/agent/chat_completion_helpers.py +++ b/agent/chat_completion_helpers.py @@ -2383,7 +2383,7 @@ def _managed_summary_call(request, callback, *, retry_count: int): final_response = "I reached the iteration limit and couldn't generate a summary." except Exception as e: - logger.warning(f"Failed to get summary response: {e}") + logger.warning("Failed to get summary response: %s", e) final_response = f"I reached the maximum iterations ({agent.max_iterations}) but couldn't summarize. Error: {str(e)}" finally: from agent import relay_llm @@ -2424,7 +2424,7 @@ def cleanup_task_resources(agent, task_id: str) -> None: _ra().cleanup_vm(task_id) except Exception as e: if agent.verbose_logging: - logger.warning(f"Failed to cleanup VM for task {task_id}: {e}") + logger.warning("Failed to cleanup VM for task %s: %s", task_id, e) try: headed = False try: @@ -2442,7 +2442,7 @@ def cleanup_task_resources(agent, task_id: str) -> None: _ra().cleanup_browser(task_id) except Exception as e: if agent.verbose_logging: - logger.warning(f"Failed to cleanup browser for task {task_id}: {e}") + logger.warning("Failed to cleanup browser for task %s: %s", task_id, e) def _build_partial_stream_stub( diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index 68453df309f7..1884c06489d9 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -2672,7 +2672,7 @@ def _perform_api_call(next_api_kwargs): # Terminal — flush buffered retry trace so user sees what happened. agent._flush_status_buffer() agent._emit_status(f"❌ Max retries ({max_retries}) exceeded for invalid responses. Giving up.") - logger.error(f"{agent.log_prefix}Invalid API response after {max_retries} retries.") + logger.error("%sInvalid API response after %d retries.", agent.log_prefix, max_retries) agent._persist_session(messages, conversation_history) _final_response = f"Invalid API response after {max_retries} retries: {_failure_hint}" return { @@ -2687,7 +2687,7 @@ def _perform_api_call(next_api_kwargs): # Backoff before retry — jittered exponential: 5s base, 120s cap wait_time = jittered_backoff(retry_count, base_delay=5.0, max_delay=120.0) agent._buffer_vprint(f"⏳ Retrying in {wait_time:.1f}s ({_failure_hint})...") - logger.warning(f"Invalid API response (retry {retry_count}/{max_retries}): {', '.join(error_details)} | Provider: {provider_name}") + logger.warning("Invalid API response (retry %d/%d): %s | Provider: %s", retry_count, max_retries, ', '.join(error_details), provider_name) # Sleep in small increments to stay responsive to interrupts sleep_end = time.time() + wait_time @@ -4599,7 +4599,7 @@ def _perform_api_call(next_api_kwargs): agent._flush_status_buffer() agent._vprint(f"{agent.log_prefix}❌ Max compression attempts ({max_compression_attempts}) reached for payload-too-large error.", force=True) agent._vprint(f"{agent.log_prefix} 💡 Try /new to start a fresh conversation, or /compress to retry compression.", force=True) - logger.error(f"{agent.log_prefix}413 compression failed after {max_compression_attempts} attempts.") + logger.error("%s413 compression failed after %d attempts.", agent.log_prefix, max_compression_attempts) agent._persist_session(messages, conversation_history) _final_response = f"Request payload too large: max compression attempts ({max_compression_attempts}) reached." return { @@ -4668,7 +4668,7 @@ def _perform_api_call(next_api_kwargs): agent._flush_status_buffer() agent._vprint(f"{agent.log_prefix}❌ Payload too large and cannot compress further.", force=True) agent._vprint(f"{agent.log_prefix} 💡 Try /new to start a fresh conversation, or /compress to retry compression.", force=True) - logger.error(f"{agent.log_prefix}413 payload too large. Cannot compress further.") + logger.error("%s413 payload too large. Cannot compress further.", agent.log_prefix) agent._persist_session(messages, conversation_history) _final_response = "Request payload too large (413). Cannot compress further." return { @@ -4741,7 +4741,7 @@ def _perform_api_call(next_api_kwargs): agent._flush_status_buffer() agent._vprint(f"{agent.log_prefix}❌ Max compression attempts ({max_compression_attempts}) reached.", force=True) agent._vprint(f"{agent.log_prefix} 💡 Try /new to start a fresh conversation, or /compress to retry compression.", force=True) - logger.error(f"{agent.log_prefix}Context compression failed after {max_compression_attempts} attempts.") + logger.error("%sContext compression failed after %d attempts.", agent.log_prefix, max_compression_attempts) agent._persist_session(messages, conversation_history) _final_response = f"Context length exceeded: max compression attempts ({max_compression_attempts}) reached." return { @@ -4860,7 +4860,7 @@ def _perform_api_call(next_api_kwargs): agent._flush_status_buffer() agent._vprint(f"{agent.log_prefix}❌ Max compression attempts ({max_compression_attempts}) reached.", force=True) agent._vprint(f"{agent.log_prefix} 💡 Try /new to start a fresh conversation, or /compress to retry compression.", force=True) - logger.error(f"{agent.log_prefix}Context compression failed after {max_compression_attempts} attempts.") + logger.error("%sContext compression failed after %d attempts.", agent.log_prefix, max_compression_attempts) agent._persist_session(messages, conversation_history) _final_response = f"Context length exceeded: max compression attempts ({max_compression_attempts}) reached." return { @@ -4918,7 +4918,7 @@ def _perform_api_call(next_api_kwargs): agent._flush_status_buffer() agent._vprint(f"{agent.log_prefix}❌ Context length exceeded and cannot compress further.", force=True) agent._vprint(f"{agent.log_prefix} 💡 The conversation has accumulated too much content. Try /new to start fresh, or /compress to manually trigger compression.", force=True) - logger.error(f"{agent.log_prefix}Context length exceeded: {new_tokens:,} tokens. Cannot compress further.") + logger.error("%sContext length exceeded: %s tokens. Cannot compress further.", agent.log_prefix, f"{new_tokens:,}") agent._persist_session(messages, conversation_history) _final_response = f"Context length exceeded ({new_tokens:,} tokens). Cannot compress further." return { @@ -5184,7 +5184,7 @@ def _perform_api_call(next_api_kwargs): f"{agent.log_prefix} for localhost, or add the server's cert to your trust store.", force=True, ) - logger.error(f"{agent.log_prefix}Non-retryable client error: {api_error}") + logger.error("%sNon-retryable client error: %s", agent.log_prefix, api_error) # Skip session persistence when the error is likely # context-overflow related (status 400 + large session). # Persisting the failed user message would make the diff --git a/agent/model_metadata.py b/agent/model_metadata.py index 6643ec820a6e..f89d25f4c8dd 100644 --- a/agent/model_metadata.py +++ b/agent/model_metadata.py @@ -1084,7 +1084,7 @@ def fetch_model_metadata(force_refresh: bool = False) -> Dict[str, Dict[str, Any return cache except Exception as e: - logger.warning(f"Failed to fetch model metadata from OpenRouter: {e}") + logger.warning("Failed to fetch model metadata from OpenRouter: %s", e) if _model_metadata_cache: return _model_metadata_cache disk_cache = _load_model_metadata_disk_cache() diff --git a/gateway/kanban_watchers.py b/gateway/kanban_watchers.py index 5fbbdea07b1e..dc2c5efd9011 100644 --- a/gateway/kanban_watchers.py +++ b/gateway/kanban_watchers.py @@ -1038,7 +1038,7 @@ async def _kanban_dispatcher_watcher(self) -> None: # Read max_spawn config to limit concurrent kanban tasks max_spawn = kanban_cfg.get("max_spawn", None) if max_spawn is not None: - logger.info(f"kanban dispatcher: max_spawn={max_spawn}") + logger.info("kanban dispatcher: max_spawn=%s", max_spawn) # Cap the number of simultaneously running tasks so slow workers # (local LLMs, resource-constrained hosts) don't pile up and time @@ -1063,7 +1063,7 @@ async def _kanban_dispatcher_watcher(self) -> None: ) max_in_progress = None else: - logger.info(f"kanban dispatcher: max_in_progress={max_in_progress}") + logger.info("kanban dispatcher: max_in_progress=%s", max_in_progress) raw_failure_limit = kanban_cfg.get("failure_limit", _kb.DEFAULT_FAILURE_LIMIT) try: diff --git a/mini_swe_runner.py b/mini_swe_runner.py index 2853abc9a01e..2994bfc6d25a 100644 --- a/mini_swe_runner.py +++ b/mini_swe_runner.py @@ -467,7 +467,7 @@ def run_task(self, task: str) -> Dict[str, Any]: response = self.client.chat.completions.create(**api_kwargs) except Exception as e: - self.logger.error(f"API call failed: {e}") + self.logger.error("API call failed: %s", e) break assistant_message = response.choices[0].message @@ -607,7 +607,7 @@ def run_batch( print(f"✅ Task {i} completed (api_calls={result['api_calls']})") except Exception as e: - self.logger.error(f"Error on task {i}: {e}") + self.logger.error("Error on task %s: %s", i, e) error_result = { "conversations": [], "completed": False, diff --git a/tools/environments/docker.py b/tools/environments/docker.py index 137d80627c3a..780d571b71f8 100644 --- a/tools/environments/docker.py +++ b/tools/environments/docker.py @@ -886,10 +886,10 @@ def __init__( self._container_name: str = "" self._image_uses_s6_init: bool = False self._all_run_args: list[str] = [] - logger.info(f"DockerEnvironment volumes: {volumes}") + logger.info("DockerEnvironment volumes: %s", volumes) # Ensure volumes is a list (config.yaml could be malformed) if volumes is not None and not isinstance(volumes, list): - logger.warning(f"docker_volumes config is not a list: {volumes!r}") + logger.warning("docker_volumes config is not a list: %r", volumes) volumes = [] # Fail fast if Docker is not available. @@ -933,7 +933,7 @@ def __init__( workspace_explicitly_mounted = False for vol in (volumes or []): if not isinstance(vol, str): - logger.warning(f"Docker volume entry is not a string: {vol!r}") + logger.warning("Docker volume entry is not a string: %r", vol) continue vol = vol.strip() if not vol: @@ -943,7 +943,7 @@ def __init__( if ":/workspace" in vol: workspace_explicitly_mounted = True else: - logger.warning(f"Docker volume '{vol}' missing colon, skipping") + logger.warning("Docker volume '%s' missing colon, skipping", vol) host_cwd_abs = os.path.abspath(os.path.expanduser(host_cwd)) if host_cwd else "" bind_host_cwd = ( @@ -953,7 +953,7 @@ def __init__( and not workspace_explicitly_mounted ) if auto_mount_cwd and host_cwd and not os.path.isdir(host_cwd_abs): - logger.debug(f"Skipping docker cwd mount: host_cwd is not a valid directory: {host_cwd}") + logger.debug("Skipping docker cwd mount: host_cwd is not a valid directory: %s", host_cwd) self._workspace_dir: Optional[str] = None self._home_dir: Optional[str] = None @@ -982,7 +982,7 @@ def __init__( ]) if bind_host_cwd: - logger.info(f"Mounting configured host cwd to /workspace: {host_cwd_abs}") + logger.info("Mounting configured host cwd to /workspace: %s", host_cwd_abs) volume_args = ["-v", f"{host_cwd_abs}:/workspace", *volume_args] elif workspace_explicitly_mounted: logger.debug("Skipping docker cwd mount: /workspace already mounted by user config") @@ -1298,7 +1298,7 @@ def __init__( run_exec=image_uses_s6_init, ) - logger.info(f"Docker volume_args: {volume_args}") + logger.info("Docker volume_args: %s", volume_args) # User-supplied extra docker run flags (docker_extra_args in config.yaml). # Appended last so they can override defaults if needed. validated_extra = [] @@ -1336,7 +1336,7 @@ def __init__( + env_args + validated_extra ) - logger.info(f"Docker run_args: {all_run_args}") + logger.info("Docker run_args: %s", all_run_args) # Start the container directly via `docker run -d`. container_name = f"hermes-{uuid.uuid4().hex[:8]}" @@ -1465,7 +1465,7 @@ def __init__( image, "sleep", "infinity", # no fixed lifetime — idle reaper handles cleanup ] - logger.debug(f"Starting container: {' '.join(run_cmd)}") + logger.debug("Starting container: %s", ' '.join(run_cmd)) try: result = subprocess.run( run_cmd, @@ -1494,7 +1494,7 @@ def __init__( ) raise self._container_id = result.stdout.strip() - logger.info(f"Started container {container_name} ({self._container_id[:12]})") + logger.info("Started container %s (%s)", container_name, self._container_id[:12]) # Build the init-time env forwarding args used to seed the snapshot. self._init_env_args = self._build_init_env_args() diff --git a/tools/skills_hub.py b/tools/skills_hub.py index 56964c132561..0316fee9d04b 100644 --- a/tools/skills_hub.py +++ b/tools/skills_hub.py @@ -464,7 +464,7 @@ def _try_github_app(self) -> Optional[str]: if resp.status_code == 201: return resp.json().get("token") except Exception as e: - logger.debug(f"GitHub App auth failed: {e}") + logger.debug("GitHub App auth failed: %s", e) return None @@ -625,7 +625,7 @@ def search(self, query: str, limit: int = 10) -> List[SkillMeta]: if query_lower in searchable: results.append(skill) except Exception as e: - logger.debug(f"Failed to search {tap['repo']}: {e}") + logger.debug("Failed to search %s: %s", tap['repo'], e) continue # Deduplicate by identifier, preferring higher trust levels. diff --git a/trajectory_compressor.py b/trajectory_compressor.py index 83248983a2ca..8feb4a5157cc 100644 --- a/trajectory_compressor.py +++ b/trajectory_compressor.py @@ -663,7 +663,7 @@ def _generate_summary(self, content: str, metrics: TrajectoryMetrics) -> str: except Exception as e: metrics.summarization_errors += 1 - self.logger.warning(f"Summarization attempt {attempt + 1} failed: {e}") + self.logger.warning("Summarization attempt %d failed: %s", attempt + 1, e) if attempt < self.config.max_retries - 1: time.sleep(jittered_backoff(attempt + 1, base_delay=self.config.retry_delay, max_delay=30.0)) @@ -732,7 +732,7 @@ async def _generate_summary_async(self, content: str, metrics: TrajectoryMetrics except Exception as e: metrics.summarization_errors += 1 - self.logger.warning(f"Summarization attempt {attempt + 1} failed: {e}") + self.logger.warning("Summarization attempt %d failed: %s", attempt + 1, e) if attempt < self.config.max_retries - 1: await asyncio.sleep(jittered_backoff(attempt + 1, base_delay=self.config.retry_delay, max_delay=30.0)) @@ -1087,7 +1087,7 @@ async def _process_directory_async(self, input_dir: Path, output_dir: Path): jsonl_files = sorted(input_dir.glob("*.jsonl")) if not jsonl_files: - self.logger.warning(f"No JSONL files found in {input_dir}") + self.logger.warning("No JSONL files found in %s", input_dir) return # Load ALL entries from all files @@ -1103,7 +1103,7 @@ async def _process_directory_async(self, input_dir: Path, output_dir: Path): entry = json.loads(line) all_entries.append((file_path, line_num, entry)) except json.JSONDecodeError as e: - self.logger.warning(f"Skipping invalid JSON at {file_path}:{line_num}: {e}") + self.logger.warning("Skipping invalid JSON at %s:%s: %s", file_path, line_num, e) total_entries = len(all_entries) @@ -1172,7 +1172,7 @@ async def process_single(file_path: Path, entry_idx: int, entry: Dict, ) except asyncio.TimeoutError: - self.logger.warning(f"Timeout processing entry from {file_path}:{entry_idx} (>{self.config.per_trajectory_timeout}s)") + self.logger.warning("Timeout processing entry from %s:%s (>%ss)", file_path, entry_idx, self.config.per_trajectory_timeout) async with progress_lock: self.aggregate_metrics.trajectories_failed += 1 @@ -1188,7 +1188,7 @@ async def process_single(file_path: Path, entry_idx: int, entry: Dict, results[file_path][entry_idx] = None except Exception as e: - self.logger.error(f"Error processing entry from {file_path}:{entry_idx}: {e}") + self.logger.error("Error processing entry from %s:%s: %s", file_path, entry_idx, e) async with progress_lock: self.aggregate_metrics.trajectories_failed += 1 From d5cf89f3b275a6c5c0e86d497f1746f86fa9de42 Mon Sep 17 00:00:00 2001 From: annguyenNous Date: Sun, 2 Aug 2026 23:01:18 +0530 Subject: [PATCH 63/69] fix(tool-executor): use lazy logging format in debug calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-applied from #50508 onto current main: the executor moved from tools/ to agent/ and the JSON-error site was refactored away into _parse_tool_arguments, but these 8 f-string debug calls survived the move verbatim. Lazy %s formatting skips string interpolation when DEBUG is off — the Tool-result line interpolates the FULL tool output (can be 100KB+) on every tool call otherwise. --- agent/tool_executor.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/agent/tool_executor.py b/agent/tool_executor.py index 6428d1ea19b8..c422c118e727 100644 --- a/agent/tool_executor.py +++ b/agent/tool_executor.py @@ -1191,8 +1191,8 @@ def _execute(next_args: dict[str, Any]) -> Any: logging.debug("file-mutation verifier record failed: %s", _ver_err) if agent.verbose_logging: - logging.debug(f"Tool {function_name} completed in {tool_duration:.2f}s") - logging.debug(f"Tool result ({len(function_result)} chars): {function_result}") + logging.debug("Tool %s completed in %.2fs", function_name, tool_duration) + logging.debug("Tool result (%d chars): %s", len(function_result), function_result) agent._current_tool = None _status_suffix = " (error)" if is_error else "" @@ -1251,7 +1251,7 @@ def _execute(next_args: dict[str, Any]) -> Any: result=display_function_result, ) except Exception as cb_err: - logging.debug(f"Tool progress callback error: {cb_err}") + logging.debug("Tool progress callback error: %s", cb_err) # Print cute message per tool if agent._should_emit_quiet_tool_messages(): @@ -1275,7 +1275,7 @@ def _execute(next_args: dict[str, Any]) -> Any: tc.id, name, display_args, display_function_result, ) except Exception as cb_err: - logging.debug(f"Tool complete callback error: {cb_err}") + logging.debug("Tool complete callback error: %s", cb_err) if ( risk_metadata is not None @@ -1899,9 +1899,9 @@ def _execute(next_args: dict) -> Any: agent._touch_activity(f"tool completed: {function_name} ({tool_duration:.1f}s){_status_suffix}") if agent.verbose_logging: - logging.debug(f"Tool {function_name} completed in {tool_duration:.2f}s") + logging.debug("Tool %s completed in %.2fs", function_name, tool_duration) _log_result = _multimodal_text_summary(function_result) - logging.debug(f"Tool result ({len(_log_result)} chars): {_log_result}") + logging.debug("Tool result (%d chars): %s", len(_log_result), _log_result) display_function_result = function_result function_result = maybe_persist_tool_result( @@ -1943,7 +1943,7 @@ def _execute(next_args: dict) -> Any: result=display_function_result, ) except Exception as cb_err: - logging.debug(f"Tool progress callback error: {cb_err}") + logging.debug("Tool progress callback error: %s", cb_err) if not _execution_blocked and agent.tool_complete_callback: try: @@ -1958,7 +1958,7 @@ def _execute(next_args: dict) -> Any: display_function_result, ) except Exception as cb_err: - logging.debug(f"Tool complete callback error: {cb_err}") + logging.debug("Tool complete callback error: %s", cb_err) if ( risk_metadata is not None From 2a2e6ee2a4bea45a9978deae81a3dce366198085 Mon Sep 17 00:00:00 2001 From: RelaxJonh <92573950+RelaxJonh@users.noreply.github.com> Date: Sun, 2 Aug 2026 23:13:12 +0530 Subject: [PATCH 64/69] fix(gateway): reap orphans on stop_profile_gateway (#75936) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit stop_profile_gateway() only killed the single PID recorded in the pid file. On repeated restarts, the new process overwrites the pid file before the old one exits, making older gateway instances invisible to subsequent stops. Each restart stacked another orphan — observed as N threads and N replies per Discord message. After killing the recorded PID, also call _reap_unsupervised_gateway_orphans() to sweep any remaining gateway processes for this profile. The reap function already handles the no-systemd guard and SIGTERM+SIGKILL escalation. Fixes #75936 --- hermes_cli/gateway.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/hermes_cli/gateway.py b/hermes_cli/gateway.py index 7d30531cf8df..aea24118a843 100644 --- a/hermes_cli/gateway.py +++ b/hermes_cli/gateway.py @@ -1583,6 +1583,12 @@ def stop_profile_gateway() -> bool: a live orphan still holds the webhook port. In that case fall back to the orphan-aware process scan so the replacement reaps the prior instance instead of stacking a duplicate on the same port (#51325). + + Even when the pid file is valid and points to the current gateway, older + orphans may linger from prior restarts that overwrote the pid file before + the old process exited. After killing the recorded PID, also sweep for + any remaining orphans so each restart produces at most one live gateway + (#75936). """ try: from gateway.status import get_running_pid, remove_pid_file @@ -1620,6 +1626,14 @@ def stop_profile_gateway() -> bool: if get_running_pid() is None: remove_pid_file() + + # Also reap any orphans from prior restarts whose PIDs were overwritten + # in the pid file before they exited (#75936). + try: + _reap_unsupervised_gateway_orphans() + except Exception: + pass + return True From fe9bdd17e305b54b826977013f70629af59f8a77 Mon Sep 17 00:00:00 2001 From: Kshitij Kothari <82637225+kshitijk4poor@users.noreply.github.com> Date: Sun, 2 Aug 2026 23:14:16 +0530 Subject: [PATCH 65/69] fix: exclude killed PID from orphan sweep, fix test, add regression test - Pass extra_exclude={pid} to _reap_unsupervised_gateway_orphans so the killed PID isn't double-killed during the sweep (#75936). - Add extra_exclude param to _reap_unsupervised_gateway_orphans signature. - Replace bare except:pass with logger.debug for diagnosability. - Fix existing test (mock _reap_unsupervised_gateway_orphans so it doesn't scan real processes and trigger conftest live-system guard). - Add regression test asserting the killed PID is excluded from the sweep. --- hermes_cli/gateway.py | 18 ++++++++++++----- tests/hermes_cli/test_gateway.py | 33 +++++++++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/hermes_cli/gateway.py b/hermes_cli/gateway.py index aea24118a843..681ce09c9ef1 100644 --- a/hermes_cli/gateway.py +++ b/hermes_cli/gateway.py @@ -1504,7 +1504,7 @@ def kill_gateway_processes( return killed -def _reap_unsupervised_gateway_orphans() -> bool: +def _reap_unsupervised_gateway_orphans(extra_exclude: set | None = None) -> bool: """Kill no-supervisor gateway orphans the pidfile/runtime record can't see. On WSL/no-systemd hosts the manual restart fallback runs the gateway @@ -1517,6 +1517,10 @@ def _reap_unsupervised_gateway_orphans() -> bool: running gateway — gating on ``supports_systemd_services()`` keeps the orphan-aware scan from killing live management processes there. + Args: + extra_exclude: Additional PIDs to skip (e.g. a PID already killed by + the caller so the sweep doesn't send a redundant SIGTERM/SIGKILL). + Returns True if at least one orphan was reaped. """ try: @@ -1528,6 +1532,8 @@ def _reap_unsupervised_gateway_orphans() -> bool: from gateway.status import _pid_exists, write_planned_stop_marker own = {os.getpid()} + if extra_exclude: + own |= extra_exclude try: # find_gateway_pids() includes no-supervisor `gateway restart` runtimes # for the current profile when no systemd supervisor is present. @@ -1628,11 +1634,13 @@ def stop_profile_gateway() -> bool: remove_pid_file() # Also reap any orphans from prior restarts whose PIDs were overwritten - # in the pid file before they exited (#75936). + # in the pid file before they exited (#75936). Exclude the PID we just + # killed so the sweep doesn't double-kill a process that's still tearing + # down — _reap_unsupervised_gateway_orphans already excludes our own PID. try: - _reap_unsupervised_gateway_orphans() - except Exception: - pass + _reap_unsupervised_gateway_orphans(extra_exclude={pid} if pid else None) + except Exception as exc: + logger.debug("orphan reap after stop_profile_gateway failed: %s", exc) return True diff --git a/tests/hermes_cli/test_gateway.py b/tests/hermes_cli/test_gateway.py index f6ffd3d51ffa..3d66de14b896 100644 --- a/tests/hermes_cli/test_gateway.py +++ b/tests/hermes_cli/test_gateway.py @@ -369,7 +369,7 @@ def test_kill_gateway_processes_force_uses_helper(self, monkeypatch): class TestStopProfileGateway: def test_stop_profile_gateway_keeps_pid_file_when_process_still_running(self, monkeypatch): - calls = {"kill": 0, "alive_probes": 0, "remove": 0} + calls = {"kill": 0, "alive_probes": 0, "remove": 0, "reap_calls": 0} monkeypatch.setattr("gateway.status.get_running_pid", lambda: 12345) # Post-#21561: the stop loop sends one SIGTERM via ``os.kill`` then @@ -389,11 +389,42 @@ def test_stop_profile_gateway_keeps_pid_file_when_process_still_running(self, mo "gateway.status.remove_pid_file", lambda: calls.__setitem__("remove", calls["remove"] + 1), ) + # Mock the orphan reap so it doesn't scan for real gateway processes + # (#75936 — stop_profile_gateway now calls _reap_unsupervised_gateway_orphans + # after killing the pid-file PID). + monkeypatch.setattr( + gateway, + "_reap_unsupervised_gateway_orphans", + lambda extra_exclude=None: calls.__setitem__("reap_calls", calls["reap_calls"] + 1) or False, + ) assert gateway.stop_profile_gateway() is True assert calls["kill"] == 1 # one SIGTERM assert calls["alive_probes"] == 20 # 20 liveness polls over the 2s window assert calls["remove"] == 0 + assert calls["reap_calls"] == 1 # orphan sweep ran after kill + + def test_stop_profile_gateway_excludes_killed_pid_from_orphan_reap(self, monkeypatch): + """The PID we killed must be excluded from the orphan sweep (#75936).""" + killed_pid = 99999 + reap_extra_excludes = [] + + monkeypatch.setattr("gateway.status.get_running_pid", lambda: killed_pid) + monkeypatch.setattr(gateway.os, "kill", lambda pid, sig: None) + monkeypatch.setattr("gateway.status._pid_exists", lambda pid: False) + monkeypatch.setattr("time.sleep", lambda _: None) + monkeypatch.setattr("gateway.status.remove_pid_file", lambda: None) + + def fake_reap(extra_exclude=None): + if extra_exclude: + reap_extra_excludes.append(extra_exclude) + return False + + monkeypatch.setattr(gateway, "_reap_unsupervised_gateway_orphans", fake_reap) + + assert gateway.stop_profile_gateway() is True + assert len(reap_extra_excludes) == 1 + assert killed_pid in reap_extra_excludes[0] def test_module_has_logger(): From 579672d87df1875ddd719ea45252af974cccc59f Mon Sep 17 00:00:00 2001 From: wayne1992127 Date: Mon, 27 Jul 2026 23:20:40 +0800 Subject: [PATCH 66/69] fix(desktop): keep cold gateway config off event loop --- hermes_cli/web_server.py | 23 ++++++++++++++------ tests/hermes_cli/test_web_server.py | 33 +++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index 857f89605ba2..5af536fdce13 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -2953,6 +2953,20 @@ def _collect_profile_gateway_topology_cached() -> Dict[str, Any]: return data +def _load_configured_gateway_platforms() -> set[str]: + """Load connected platform names away from the asyncio event loop. + + The first ``load_gateway_config()`` call performs platform discovery and + can take longer than Desktop's WebSocket connect timeout on Windows. This + helper is synchronous by design; ``get_status`` runs it in Starlette's + worker pool so a concurrent ``/api/ws`` handshake can still complete. + """ + from gateway.config import load_gateway_config + + gateway_config = load_gateway_config() + return {platform.value for platform in gateway_config.get_connected_platforms()} + + @app.get("/api/ssh/ownership") async def get_ssh_ownership(request: Request): _require_token(request) @@ -3054,12 +3068,9 @@ def _bounded_health_probe(): gateway_updated_at = None configured_gateway_platforms: set[str] | None = None try: - from gateway.config import load_gateway_config - - gateway_config = load_gateway_config() - configured_gateway_platforms = { - platform.value for platform in gateway_config.get_connected_platforms() - } + configured_gateway_platforms = await run_in_threadpool( + _load_configured_gateway_platforms + ) except Exception: configured_gateway_platforms = None diff --git a/tests/hermes_cli/test_web_server.py b/tests/hermes_cli/test_web_server.py index 2a201966cab4..c7f3c5fc85da 100644 --- a/tests/hermes_cli/test_web_server.py +++ b/tests/hermes_cli/test_web_server.py @@ -318,6 +318,39 @@ def test_get_sessions_poll_preserves_pending_wal(self): monitor.close() writer.close() + def test_get_status_loads_gateway_config_off_event_loop(self, monkeypatch): + """Cold gateway config loading must not block the WebSocket loop. + + On Windows the first ``load_gateway_config()`` call imports and + discovers platform adapters and can take longer than Desktop's 15s + WebSocket timeout. Running it inline makes a concurrent /api/ws + handshake time out before ``gateway.ready`` can be sent. + """ + import gateway.config as gateway_config + import hermes_cli.web_server as web_server + + seen = {} + + class _Config: + @staticmethod + def get_connected_platforms(): + return [] + + def _load(): + seen["thread"] = threading.get_ident() + return _Config() + + monkeypatch.setattr(gateway_config, "load_gateway_config", _load) + + async def _run(): + event_loop_thread = threading.get_ident() + await web_server.get_status() + return event_loop_thread + + event_loop_thread = asyncio.run(_run()) + + assert seen["thread"] != event_loop_thread + def test_get_sessions_auto_archive_uses_maintenance_writer(self): from hermes_cli import web_server from hermes_cli.config import load_config, save_config From 70a3c2d9c9aa460d82721fb597625c0e61071a93 Mon Sep 17 00:00:00 2001 From: Jesse Casco Date: Mon, 20 Jul 2026 13:20:01 -0400 Subject: [PATCH 67/69] perf(discord): stream voice PCM to ffmpeg instead of a temp file pcm_to_wav staged every captured utterance in a NamedTemporaryFile just to hand ffmpeg an input path, then unlinked it. Feed the PCM to ffmpeg's stdin instead: one fewer file created, written, read back and removed per voice utterance, and the try/finally cleanup goes away with it. The WAV output deliberately still goes to output_path rather than being captured from stdout. ffmpeg cannot seek on a pipe, so a piped WAV is written with placeholder 0xFFFFFFFF RIFF/data chunk sizes -- Python's wave module then reports 2147483647 frames for a 1s clip, and strict readers misjudge the length. Writing to the real path lets ffmpeg seek back and patch the header. Tests cover both halves: that the PCM goes over stdin with no temp file, and (when ffmpeg is installed) that the resulting header reports the true frame count. Co-Authored-By: Claude Opus 4.8 --- plugins/platforms/discord/adapter.py | 52 +++++++++++++--------------- tests/gateway/test_voice_command.py | 51 +++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 27 deletions(-) diff --git a/plugins/platforms/discord/adapter.py b/plugins/platforms/discord/adapter.py index 88d87bc8b35c..4033feca90b7 100644 --- a/plugins/platforms/discord/adapter.py +++ b/plugins/platforms/discord/adapter.py @@ -838,34 +838,32 @@ def flush_pending(self) -> list: @staticmethod def pcm_to_wav(pcm_data: bytes, output_path: str, src_rate: int = 48000, src_channels: int = 2): - """Convert raw PCM to 16kHz mono WAV via ffmpeg.""" - with tempfile.NamedTemporaryFile(suffix=".pcm", delete=False) as f: - f.write(pcm_data) - pcm_path = f.name - try: - from hermes_cli._subprocess_compat import windows_hide_flags + """Convert raw PCM to 16kHz mono WAV via ffmpeg. - subprocess.run( - [ - resolve_ffmpeg_executable(), "-y", "-loglevel", "error", - "-f", "s16le", - "-ar", str(src_rate), - "-ac", str(src_channels), - "-i", pcm_path, - "-ar", "16000", - "-ac", "1", - output_path, - ], - check=True, - timeout=10, - stdin=subprocess.DEVNULL, - creationflags=windows_hide_flags(), - ) - finally: - try: - os.unlink(pcm_path) - except OSError: - pass + The PCM is fed straight to ffmpeg's stdin, which avoids staging it in a + temp file on every utterance. The WAV is still written to *output_path* + rather than captured from stdout: ffmpeg cannot seek on a pipe, so a + piped WAV carries placeholder (0xFFFFFFFF) RIFF/data sizes that make + strict readers misreport the length. + """ + from hermes_cli._subprocess_compat import windows_hide_flags + + subprocess.run( + [ + resolve_ffmpeg_executable(), "-y", "-loglevel", "error", + "-f", "s16le", + "-ar", str(src_rate), + "-ac", str(src_channels), + "-i", "pipe:0", + "-ar", "16000", + "-ac", "1", + output_path, + ], + input=pcm_data, + check=True, + timeout=10, + creationflags=windows_hide_flags(), + ) def _read_dm_role_auth_guild() -> Optional[int]: diff --git a/tests/gateway/test_voice_command.py b/tests/gateway/test_voice_command.py index b2d4ab6832a8..4a988a6e036a 100644 --- a/tests/gateway/test_voice_command.py +++ b/tests/gateway/test_voice_command.py @@ -1947,3 +1947,54 @@ def _fake_play(path): ) # And the temp file is cleaned up afterwards. assert not os.path.exists(played[0]), "temp WAV was not unlinked" + + +class TestPcmToWav: + """pcm_to_wav streams PCM through ffmpeg's stdin, not a temp file.""" + + def test_pcm_is_piped_to_stdin_not_staged_on_disk(self, tmp_path): + from plugins.platforms.discord.adapter import VoiceReceiver + + out = tmp_path / "out.wav" + with patch("plugins.platforms.discord.adapter.subprocess.run") as run: + VoiceReceiver.pcm_to_wav(b"\x00\x01" * 16, str(out)) + + args, kwargs = run.call_args + cmd = args[0] + assert kwargs["input"] == b"\x00\x01" * 16, "PCM must be fed via stdin" + assert "pipe:0" in cmd, "ffmpeg must read the PCM from stdin" + assert cmd[-1] == str(out), ( + "the WAV must be written to the real path; ffmpeg cannot seek on a " + "pipe, so a piped WAV gets placeholder RIFF/data sizes" + ) + assert not any(str(a).endswith(".pcm") for a in cmd), ( + "no temp .pcm file should be staged" + ) + + @pytest.mark.skipif( + __import__("shutil").which("ffmpeg") is None, reason="ffmpeg not installed", + ) + def test_output_wav_header_reports_true_length(self, tmp_path): + """A piped-stdout WAV reports 0xFFFFFFFF sizes; the written file must not.""" + import math + import struct + import wave + + from plugins.platforms.discord.adapter import VoiceReceiver + + frames = 48000 # 1s @ 48kHz stereo + pcm = b"".join( + struct.pack(" 16kHz is a 3x decimation of a 1s clip. + assert w.getnframes() == 16000 From c2ff2e8b17f5dd0460aa020aaa21deb59d7fe15f Mon Sep 17 00:00:00 2001 From: kshitij <82637225+kshitijk4poor@users.noreply.github.com> Date: Sun, 2 Aug 2026 23:17:21 +0530 Subject: [PATCH 68/69] polish(discord): surface ffmpeg stderr on pcm_to_wav failure Simplify-pass follow-up on the #68157 salvage (regression-neutral \u2014 the old temp-file version was equally bare): capture ffmpeg's -loglevel error output so a CalledProcessError carries the real message, and log it at the voice-input catch site. Parity with transcription_tools' ffmpeg call sites. Live-verified: forced ffmpeg failure produces the captured message in the exception. --- plugins/platforms/discord/adapter.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/plugins/platforms/discord/adapter.py b/plugins/platforms/discord/adapter.py index 4033feca90b7..99e0f9a2f57b 100644 --- a/plugins/platforms/discord/adapter.py +++ b/plugins/platforms/discord/adapter.py @@ -862,6 +862,11 @@ def pcm_to_wav(pcm_data: bytes, output_path: str, input=pcm_data, check=True, timeout=10, + # Capture ffmpeg's -loglevel error output so a failure's + # CalledProcessError carries the actual message (parity with + # tools/transcription_tools' ffmpeg call sites) instead of + # "returned non-zero exit status N" with stderr detached. + stderr=subprocess.PIPE, creationflags=windows_hide_flags(), ) @@ -4479,7 +4484,18 @@ async def _process_voice_input(self, guild_id: int, user_id: int, pcm_data: byte transcript=transcript, ) except Exception as e: - logger.warning("Voice input processing failed: %s", e, exc_info=True) + # CalledProcessError from pcm_to_wav carries ffmpeg's captured + # stderr — surface it, or the log only says "exit status N". + _ff_err = getattr(e, "stderr", None) + if _ff_err: + if isinstance(_ff_err, bytes): + _ff_err = _ff_err.decode("utf-8", "replace") + logger.warning( + "Voice input processing failed: %s (ffmpeg: %s)", + e, _ff_err.strip(), exc_info=True, + ) + else: + logger.warning("Voice input processing failed: %s", e, exc_info=True) finally: try: os.unlink(wav_path) From 4092ca57272ca88d9a7f25dd8460f1ab98688e9c Mon Sep 17 00:00:00 2001 From: spfcraze Date: Sun, 2 Aug 2026 12:43:55 -0400 Subject: [PATCH 69/69] test(tools): pin the WSL2 PowerShell TTS fallback in voice tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The WSL voice-detection tests assert that a no-forwarding WSL environment (no PULSE_SERVER, no PIPEWIRE_REMOTE) hard-blocks voice mode. But the WSL2 PowerShell TTS fallback (added later) relaxes that block when powershell.exe + ffmpeg exist on the host: detect_audio_environment() downgrades to a notice and reports available=True. The tests never pinned _wsl_powershell_tts_available, so their pass/fail depended on the host environment — a WSL machine with powershell.exe and ffmpeg (both present by default in WSL2) failed test_wsl_without_forwarding_ still_blocks and test_wsl_without_pulse_blocks_voice. Deterministic unit tests must not depend on host binaries. Pin the fallback to False in the no-forwarding tests (they test the hard-block path) and add an explicit test for the fallback path (available True with a WSL notice) so that behavior is covered deterministically. --- tests/tools/test_voice_mode.py | 5 +++++ tests/tools/test_voice_wsl_pipewire.py | 25 +++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/tests/tools/test_voice_mode.py b/tests/tools/test_voice_mode.py index 51d921d61342..2d693a7f6ef5 100644 --- a/tests/tools/test_voice_mode.py +++ b/tests/tools/test_voice_mode.py @@ -207,6 +207,11 @@ def test_wsl_without_pulse_blocks_voice(self, monkeypatch, tmp_path): monkeypatch.setattr("tools.voice_mode._pulse_socket_reachable", lambda: False) monkeypatch.setattr("tools.voice_mode._import_audio", lambda: (MagicMock(), MagicMock())) + # Pin the WSL2 PowerShell TTS fallback: without this the no-forwarding + # block depends on whether powershell.exe + ffmpeg exist on the host + # (a WSL machine with both sees available=True via the fallback and + # this test flakes). The fallback path is covered by its own test. + monkeypatch.setattr("tools.voice_mode._wsl_powershell_tts_available", lambda: False) proc_version = tmp_path / "proc_version" proc_version.write_text("Linux 5.15.0-microsoft-standard-WSL2") diff --git a/tests/tools/test_voice_wsl_pipewire.py b/tests/tools/test_voice_wsl_pipewire.py index 395d401fbf4e..ff82e7f208c3 100644 --- a/tests/tools/test_voice_wsl_pipewire.py +++ b/tests/tools/test_voice_wsl_pipewire.py @@ -28,6 +28,11 @@ def _base(monkeypatch): monkeypatch.delenv("PIPEWIRE_REMOTE", raising=False) monkeypatch.setattr("hermes_constants.is_container", lambda: False) monkeypatch.setattr("tools.voice_mode._pulse_socket_reachable", lambda: False) + # The WSL2 PowerShell TTS fallback makes the no-forwarding path host- + # dependent (powershell.exe + ffmpeg on PATH). Pin it so these tests are + # deterministic on any machine: the fallback is only exercised by the + # dedicated test that mocks it to True. + monkeypatch.setattr("tools.voice_mode._wsl_powershell_tts_available", lambda: False) sd = MagicMock(); sd.query_devices.return_value = [{"name": "dev"}] monkeypatch.setattr("tools.voice_mode._import_audio", lambda: (sd, MagicMock())) @@ -56,3 +61,23 @@ def test_wsl_without_forwarding_still_blocks(monkeypatch): res = detect_audio_environment() assert res["available"] is False assert any("WSL" in w for w in res["warnings"]) + + +def test_wsl_without_forwarding_but_powershell_fallback_allows_tts(monkeypatch): + """The WSL2 PowerShell TTS fallback relaxes the no-forwarding block for + OUTPUT (TTS playback) while keeping recording guidance visible. + + Regression for the stale-test bug: this path was added by the PowerShell + fallback feature but never unit-tested, so a WSL host with powershell.exe + and ffmpeg (the fallback's only preconditions) flipped the no-forwarding + test from pass to fail depending on the machine. + """ + _base(monkeypatch) + _force_wsl(monkeypatch) # no PULSE_SERVER, no PIPEWIRE_REMOTE + from tools.voice_mode import detect_audio_environment + monkeypatch.setattr("tools.voice_mode._wsl_powershell_tts_available", lambda: True) + res = detect_audio_environment() + # TTS playback works via Media.SoundPlayer on the Windows host, so voice + # mode is available — but the WSL guidance notice must still surface. + assert res["available"] is True + assert any("WSL" in w for w in res["notices"])