diff --git a/acp_adapter/server.py b/acp_adapter/server.py index f6e0462ec9d1..46577c4fa529 100644 --- a/acp_adapter/server.py +++ b/acp_adapter/server.py @@ -78,6 +78,7 @@ COMPRESSED_SUMMARY_METADATA_KEY, ContextCompressor, ) +from agent.interrupt_compat import request_hard_interrupt from tools.approval import ( reset_hermes_interactive_context, set_hermes_interactive_context, @@ -1547,8 +1548,8 @@ async def cancel(self, session_id: str, **kwargs: Any) -> None: # redirectable work. state.cancel_event.set() try: - if getattr(state, "agent", None) and hasattr(state.agent, "interrupt"): - state.agent.interrupt() + if getattr(state, "agent", None): + request_hard_interrupt(state.agent) except Exception: logger.debug( "Failed to interrupt ACP session %s", diff --git a/agent/agent_init.py b/agent/agent_init.py index d36d1607a897..68b41a231b39 100644 --- a/agent/agent_init.py +++ b/agent/agent_init.py @@ -765,6 +765,9 @@ def init_agent( # Interrupt mechanism for breaking out of tool loops agent._interrupt_requested = False agent._interrupt_message = None # Optional message that triggered interrupt + # Explicit hard cancellation is separate from redirect/message state. A + # thread-safe Event makes the cause atomic for auxiliary stream pollers. + agent._hard_interrupt_requested = threading.Event() agent._execution_thread_id: int | None = None # Set at run_conversation() start agent._interrupt_thread_signal_pending = False agent._client_lock = threading.RLock() diff --git a/agent/agent_runtime_helpers.py b/agent/agent_runtime_helpers.py index 12bed0c666fd..579314018cde 100644 --- a/agent/agent_runtime_helpers.py +++ b/agent/agent_runtime_helpers.py @@ -151,7 +151,7 @@ def convert_to_trajectory_format(agent, messages: List[Dict[str, Any]], user_que except json.JSONDecodeError: # This shouldn't happen since we validate and retry during conversation, # but if it does, log warning and use empty dict - logger.warning(f"Unexpected invalid JSON in trajectory conversion: {tool_call['function']['arguments'][:100]}") + logger.warning("Unexpected invalid JSON in trajectory conversion: %s", tool_call['function']['arguments'][:100]) arguments = {} tool_call_json = { @@ -1210,7 +1210,7 @@ def _rotate_failed_credential(rotate_status: int): refreshed_id, ) return False, has_retried_429 - _ra().logger.info(f"Credential auth failure — refreshed pool entry {getattr(refreshed, 'id', '?')}") + _ra().logger.info("Credential auth failure — refreshed pool entry %s", getattr(refreshed, 'id', '?')) agent._swap_credential(refreshed) return True, has_retried_429 # Refresh failed — rotate to next credential instead of giving up. @@ -1835,7 +1835,7 @@ def dump_api_request_debug( return dump_file except Exception as dump_error: if agent.verbose_logging: - logger.warning(f"Failed to dump API request debug payload: {dump_error}") + logger.warning("Failed to dump API request debug payload: %s", dump_error) return None diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index 8decf5dd22c4..22bf9c165cf0 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -228,30 +228,134 @@ def _create_openai_client(*, api_key: str, base_url: str, **kwargs: Any) -> Any: # part-way, compression falls back to a static "summary unavailable" marker # and the real handoff is lost (#23975). A thread-local flag lets such a # task mark its in-flight LLM call as interrupt-protected; the Codex -# Responses stream's cancellation check honors it. TIMEOUTS still fire +# Responses stream's cancellation check honors it. An explicit host cancel +# (CLI Ctrl+C or /stop) may install a cancel check that overrides protection; +# ordinary incoming-message interrupts remain protected. TIMEOUTS still fire # (a hung call must die), and all OTHER aux tasks (vision, web_extract, # title_generation, …) remain freely interruptible. _aux_interrupt_protection = threading.local() +class AuxiliaryExplicitCancellation(BaseException): + """Frozen signal that an auxiliary attempt was explicitly hard-cancelled. + + This deliberately follows ``asyncio.CancelledError`` and inherits directly + from ``BaseException``: provider retry/fallback code catches ``Exception`` + broadly and must never reinterpret an explicit host stop as a transport + failure. ``cause`` is immutable class data so downstream compression code + does not re-query a mutable host Event after the transport has unwound. + """ + + cause = "explicit_host_cancel" + + def __init__(self) -> None: + super().__init__("auxiliary request explicitly cancelled by host") + + def _aux_interrupt_protected() -> bool: return bool(getattr(_aux_interrupt_protection, "active", False)) +def _aux_interrupt_cancel_requested() -> bool: + """Return whether an explicit host cancel overrides aux protection.""" + event = getattr(_aux_interrupt_protection, "cancel_event", None) + if event is not None: + try: + return bool(event.is_set()) + except Exception: + logger.debug("aux interrupt cancel event check failed", exc_info=True) + return False + check = getattr(_aux_interrupt_protection, "cancel_check", None) + if not callable(check): + return False + try: + return bool(check()) + except Exception: + logger.debug("aux interrupt cancel check failed", exc_info=True) + return False + + @contextlib.contextmanager -def aux_interrupt_protection(active: bool = True): +def aux_interrupt_protection( + active: bool = True, + cancel_check=None, + cancel_event=None, +): """Mark the current thread's auxiliary LLM call as interrupt-protected. Used by atomic aux tasks (compression) so a mid-flight gateway interrupt doesn't abort the call and trigger a degraded fallback. Re-entrant-safe: - restores the previous value on exit. + restores the previous value on exit. ``cancel_check`` lets the host retain + an explicit hard-cancel path; ``cancel_event`` is preferred when the host + already owns an Event. Nested protection scopes inherit both values. """ prev = getattr(_aux_interrupt_protection, "active", False) + prev_cancel_check = getattr(_aux_interrupt_protection, "cancel_check", None) + prev_cancel_event = getattr(_aux_interrupt_protection, "cancel_event", None) _aux_interrupt_protection.active = active + if callable(cancel_check): + _aux_interrupt_protection.cancel_check = cancel_check + if cancel_event is not None and callable(getattr(cancel_event, "is_set", None)): + _aux_interrupt_protection.cancel_event = cancel_event try: yield finally: _aux_interrupt_protection.active = prev + _aux_interrupt_protection.cancel_check = prev_cancel_check + _aux_interrupt_protection.cancel_event = prev_cancel_event + + +def _capture_aux_cancel_check() -> Optional[Callable[[], Any]]: + """Capture the current explicit-cancel source on the owning request thread.""" + event = getattr(_aux_interrupt_protection, "cancel_event", None) + is_set = getattr(event, "is_set", None) + if callable(is_set): + return is_set + check = getattr(_aux_interrupt_protection, "cancel_check", None) + if callable(check): + # Preserve callable identity so attempt-local decision objects retain + # methods such as begin_timeout_cleanup() when captured by adapters. + return check + return None + + +def _captured_aux_cancel_requested(cancel_check: Callable[[], Any]) -> bool: + """Read a request-thread cancellation source without leaking its failures.""" + try: + return bool(cancel_check()) + except Exception: + logger.debug("captured aux cancel check failed", exc_info=True) + return False + + +class _AuxiliaryCancellationDecision: + """Atomically choose explicit cancellation or provider timeout per attempt.""" + + def __init__(self, source_cancel_check: Callable[[], Any]) -> None: + self._source_cancel_check = source_cancel_check + self._lock = threading.Lock() + self._outcome = "active" + + def __call__(self) -> bool: + with self._lock: + if self._outcome == "cancelled": + return True + if self._outcome == "timed_out": + return False + if _captured_aux_cancel_requested(self._source_cancel_check): + self._outcome = "cancelled" + return True + return False + + def begin_timeout_cleanup(self) -> bool: + """Return whether timeout won and destructive cleanup is permitted.""" + with self._lock: + if self._outcome == "active": + if _captured_aux_cancel_requested(self._source_cancel_check): + self._outcome = "cancelled" + else: + self._outcome = "timed_out" + return self._outcome == "timed_out" # ── Forward-progress hook for streamed auxiliary calls ─────────────────── @@ -298,6 +402,75 @@ def aux_progress_hook(hook): _aux_progress.hook = prev +def _run_protected_sync_provider_call( + callback: Callable[[dict[str, Any]], Any], + kwargs: dict[str, Any], +) -> Any: + """Run one protected provider callback in an attempt-isolated daemon. + + A hard cancel must release the compression-owning thread promptly, but + auxiliary clients are process-shared and cannot safely be closed or evicted + to wake one request. Only protected calls with a captured hard-cancel source + use this seam. Their provider callback (including stream aggregation) runs + in a daemon worker while the owner polls cancellation. On cancel the owner + unwinds immediately; the worker is left to finish under the provider timeout + already present in ``kwargs``. It owns no transcript or compressor commit + state and never holds the session lock. + + Ordinary auxiliary calls, and protected calls without a cancellation source, + retain the historical direct synchronous path with no extra thread. + """ + source_cancel_check = _capture_aux_cancel_check() + if not _aux_interrupt_protected() or not callable(source_cancel_check): + return callback(kwargs) + + # Freeze one linearized outcome for this isolated attempt. The host Event is + # reused and cleared on a later turn, while the Codex timeout Timer may race + # owner polling. Both paths must decide under the same attempt-local lock. + cancel_check = _AuxiliaryCancellationDecision(source_cancel_check) + + if cancel_check(): + raise AuxiliaryExplicitCancellation() + + progress_hook = getattr(_aux_progress, "hook", None) + provider_context = contextvars.copy_context() + done = threading.Event() + outcome: dict[str, Any] = {} + + def _provider_worker() -> None: + try: + with aux_progress_hook(progress_hook), aux_interrupt_protection( + cancel_check=cancel_check + ): + outcome["result"] = callback(kwargs) + except BaseException as exc: + outcome["exception"] = exc + finally: + done.set() + + threading.Thread( + target=provider_context.run, + args=(_provider_worker,), + name="hermes-protected-aux-provider", + daemon=True, + ).start() + + while True: + # Cancellation is checked before and after every completion wait so it + # wins whenever result publication and the host Event become visible in + # the same polling interval. + if _captured_aux_cancel_requested(cancel_check): + raise AuxiliaryExplicitCancellation() + if not done.wait(0.02): + continue + if _captured_aux_cancel_requested(cancel_check): + raise AuxiliaryExplicitCancellation() + exception = outcome.get("exception") + if exception is not None: + raise exception + return outcome.get("result") + + def _safe_isinstance(obj: Any, maybe_type: Any) -> bool: """Return False instead of raising when a patched symbol is not a type.""" try: @@ -960,6 +1133,29 @@ def _nous_min_key_ttl_seconds() -> int: return 1800 +def _scoped_key_env(name: str) -> str: + """Read a provider API key env var through the profile secret scope. + + Auxiliary-client resolution runs both inside agent turns (secret scope + installed — its verdict is authoritative under multiplex, so a scoped + miss must NOT borrow another profile's process-env key) and on unscoped + startup/CLI probe paths, which keep the legacy ``os.environ`` read via + the ``UnscopedSecretError`` fallback (Slack pattern, #59739). + """ + if not name: + return "" + try: + from agent.secret_scope import UnscopedSecretError, get_secret + + try: + return (get_secret(name) or "").strip() + except UnscopedSecretError: + pass + except Exception: + pass + return (os.getenv(name) or "").strip() + + # ── Codex Responses → chat.completions adapter ───────────────────────────── # All auxiliary consumers call client.chat.completions.create(**kwargs) and # read response.choices[0].message.content. This adapter translates those @@ -1155,12 +1351,53 @@ def create(self, **kwargs) -> Any: deadline = time.monotonic() + float(total_timeout) if total_timeout else None timed_out = threading.Event() timeout_timer: Optional[threading.Timer] = None + # A protected provider call may outlive its owning compression attempt: + # the owner returns promptly on hard cancellation while this adapter is + # still blocked in the SDK stream on its isolated worker. Timer threads + # do not inherit this worker's thread-local protection state, so freeze + # the hard-cancel source here, before creating the timer. + protected_cancel_check = ( + _capture_aux_cancel_check() if _aux_interrupt_protected() else None + ) + attempt_stream_lock = threading.Lock() + attempt_stream: List[Any] = [] def _timeout_message() -> str: return f"Codex auxiliary Responses stream exceeded {float(total_timeout):.1f}s total timeout" def _close_client_on_timeout() -> None: + begin_timeout_cleanup = getattr( + protected_cancel_check, "begin_timeout_cleanup", None + ) + if callable(begin_timeout_cleanup): + timeout_won = bool(begin_timeout_cleanup()) + else: + timeout_won = not ( + callable(protected_cancel_check) + and _captured_aux_cancel_requested(protected_cancel_check) + ) + # Publish transport timeout only after the attempt-local decision is + # fixed, so owner polling cannot observe completion in between. timed_out.set() + if not timeout_won: + # The request owner already hard-cancelled this attempt. The + # OpenAI client is process-shared, so closing/evicting it here + # would disrupt unrelated sessions. Wake only this attempt's + # event stream when responses.create() returned one in time; + # otherwise rely on the bounded SDK/provider timeout. + with attempt_stream_lock: + stream = attempt_stream[0] if attempt_stream else None + close_stream = getattr(stream, "close", None) + if callable(close_stream): + try: + close_stream() + except Exception: + logger.debug( + "Codex auxiliary: cancelled attempt stream close " + "during timeout failed", + exc_info=True, + ) + return close = getattr(self._client, "close", None) if callable(close): try: @@ -1187,11 +1424,14 @@ def _check_cancelled() -> None: from tools.interrupt import is_interrupted # Honor interrupt protection for atomic aux tasks (compression): # a mid-flight gateway interrupt must NOT abort the summary call - # and trigger a degraded fallback marker (#23975). Timeouts above - # still fire; other aux tasks remain interruptible. + # and trigger a degraded fallback marker (#23975). Explicit host + # cancellation has its own frozen exception; timeouts above still + # fire and other aux tasks remain interruptible. + if _aux_interrupt_cancel_requested(): + raise AuxiliaryExplicitCancellation() if is_interrupted() and not _aux_interrupt_protected(): raise InterruptedError("Codex auxiliary Responses stream interrupted") - except InterruptedError: + except (InterruptedError, AuxiliaryExplicitCancellation): raise except Exception: # Interrupt state is a best-effort UX hook; never make it a @@ -1230,6 +1470,25 @@ def _on_each_event(_event: Any) -> None: _check_cancelled() event_stream = self._client.responses.create(**stream_kwargs) + with attempt_stream_lock: + attempt_stream.append(event_stream) + # The timer can fire while responses.create() is blocked. If the + # cancelled attempt had no stream to close at that instant, close it + # now that it is safely attempt-owned; never touch the shared client. + if ( + timed_out.is_set() + and callable(protected_cancel_check) + and _captured_aux_cancel_requested(protected_cancel_check) + ): + close_fn = getattr(event_stream, "close", None) + if callable(close_fn): + try: + close_fn() + except Exception: + logger.debug( + "Codex auxiliary: late cancelled attempt stream close failed", + exc_info=True, + ) try: # Some Codex-compatible hosts accept ``stream=True`` but return # a completed Responses object instead of an SSE iterator. Do @@ -1251,6 +1510,8 @@ def _on_each_event(_event: Any) -> None: close_fn() except Exception: pass + with attempt_stream_lock: + attempt_stream.clear() if final is None: raise RuntimeError("Codex auxiliary Responses stream did not return a final response") @@ -2232,7 +2493,7 @@ def _try_openrouter(explicit_api_key: str = None, model: str = None) -> Tuple[Op # the OPENROUTER_API_KEY env-var path rather than failing outright. logger.debug("Auxiliary client: OpenRouter pool exhausted, trying OPENROUTER_API_KEY") - or_key = explicit_api_key or os.getenv("OPENROUTER_API_KEY") + or_key = explicit_api_key or _scoped_key_env("OPENROUTER_API_KEY") if not or_key: _mark_provider_unhealthy("openrouter", ttl=60) return None, None @@ -2249,7 +2510,7 @@ def _describe_openrouter_unavailable() -> str: return "OpenRouter credential pool has no usable entries (credentials may be exhausted)" if not _pool_runtime_api_key(entry): return "OpenRouter credential pool entry is missing a runtime API key" - if not str(os.getenv("OPENROUTER_API_KEY") or "").strip(): + if not _scoped_key_env("OPENROUTER_API_KEY"): return "OPENROUTER_API_KEY not set" return "no usable OpenRouter credentials found" @@ -2672,14 +2933,17 @@ def _relay_sync_completion( ) -> Any: callback = create or (lambda request: client.chat.completions.create(**request)) route = _relay_auxiliary_metadata(provider=provider, api_mode=api_mode) + # Protected compression calls isolate only the provider callback and stream + # aggregation. The owning thread remains free to unwind its lease/DB + # transaction on hard cancel without touching the process-shared client. if route is None: - return callback(kwargs) + return _run_protected_sync_provider_call(callback, kwargs) provider_name, fallback_model, metadata = route from agent import relay_llm return relay_llm.execute_current( kwargs, - callback, + lambda request: _run_protected_sync_provider_call(callback, request), name=provider_name, model_name=str(kwargs.get("model") or fallback_model), metadata=metadata, @@ -2878,7 +3142,7 @@ def _resolve_custom_runtime() -> Tuple[Optional[str], Optional[str], Optional[st if not isinstance(runtime, dict): openai_base = os.getenv("OPENAI_BASE_URL", "").strip().rstrip("/") - openai_key = os.getenv("OPENAI_API_KEY", "").strip() + openai_key = _scoped_key_env("OPENAI_API_KEY") if not openai_base: return None, None, None runtime = { @@ -5684,7 +5948,7 @@ def _wrap_if_needed(client_obj, final_model_str: str, base_url_str: str = "", custom_base = _to_openai_base_url(explicit_base_url).strip() custom_key = ( (explicit_api_key or "").strip() - or os.getenv("OPENAI_API_KEY", "").strip() + or _scoped_key_env("OPENAI_API_KEY") or _read_main_api_key_if_same_host(custom_base) or "no-key-required" # local servers don't need auth ) @@ -5780,7 +6044,7 @@ def _wrap_if_needed(client_obj, final_model_str: str, base_url_str: str = "", custom_key = (custom_entry.get("api_key") or "").strip() custom_key_env = (custom_entry.get("key_env") or custom_entry.get("api_key_env") or "").strip() if not custom_key and custom_key_env: - custom_key = os.getenv(custom_key_env, "").strip() + custom_key = _scoped_key_env(custom_key_env) custom_key = custom_key or "no-key-required" if custom_key == "no-key-required": logger.warning( @@ -6602,7 +6866,7 @@ def auxiliary_max_tokens_param(value: int, *, model: Optional[str] = None) -> di misses the case where a custom base URL serves e.g. ``gpt-5.4``. """ custom_base = _current_custom_base_url() - or_key = os.getenv("OPENROUTER_API_KEY") + or_key = _scoped_key_env("OPENROUTER_API_KEY") # Use max_completion_tokens for direct OpenAI-compatible providers that reject # max_tokens on newer GPT-4o/o-series/GPT-5-style models. _custom_host = base_url_hostname(custom_base) or "" @@ -7063,7 +7327,7 @@ def _resolve_task_provider_model( task_config.get("key_env") or task_config.get("api_key_env") or "" ).strip() if cfg_key_env: - cfg_api_key = os.getenv(cfg_key_env, "").strip() or None + cfg_api_key = _scoped_key_env(cfg_key_env) or None cfg_api_mode = str(task_config.get("api_mode", "")).strip() or None # 'auto' is a sentinel meaning "inherit from main runtime / auto-detect", not diff --git a/agent/azure_identity_adapter.py b/agent/azure_identity_adapter.py index 9506715019d7..dd0f62ab9737 100644 --- a/agent/azure_identity_adapter.py +++ b/agent/azure_identity_adapter.py @@ -367,11 +367,27 @@ class name. Users wanting the precise class can run with info["tenant_id_env"] = os.environ["AZURE_TENANT_ID"].strip() # Surface which env-var sources are present without minting yet. + # Credential-bearing vars (AZURE_CLIENT_SECRET, AZURE_FEDERATED_TOKEN_FILE) + # are read through the profile secret scope so a multiplexed profile's + # diagnostics don't report another profile's env-bridged credentials; + # unscoped CLI probes keep the legacy env read (Slack pattern). + def _scoped_env(name: str) -> str: + try: + from agent.secret_scope import UnscopedSecretError, get_secret + + try: + return (get_secret(name) or "").strip() + except UnscopedSecretError: + pass + except Exception: + pass + return os.environ.get(name, "").strip() + env_sources = [] - if os.environ.get("AZURE_FEDERATED_TOKEN_FILE", "").strip(): + if _scoped_env("AZURE_FEDERATED_TOKEN_FILE"): env_sources.append("WorkloadIdentityCredential (AZURE_FEDERATED_TOKEN_FILE)") if (os.environ.get("AZURE_CLIENT_ID", "").strip() - and os.environ.get("AZURE_CLIENT_SECRET", "").strip() + and _scoped_env("AZURE_CLIENT_SECRET") and os.environ.get("AZURE_TENANT_ID", "").strip()): env_sources.append("EnvironmentCredential (client secret)") if os.environ.get("IDENTITY_ENDPOINT", "").strip() or os.environ.get("MSI_ENDPOINT", "").strip(): diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.py index e40333f0e601..e6e8ab7fdc1f 100644 --- a/agent/chat_completion_helpers.py +++ b/agent/chat_completion_helpers.py @@ -2383,7 +2383,7 @@ def _managed_summary_call(request, callback, *, retry_count: int): final_response = "I reached the iteration limit and couldn't generate a summary." except Exception as e: - logger.warning(f"Failed to get summary response: {e}") + logger.warning("Failed to get summary response: %s", e) final_response = f"I reached the maximum iterations ({agent.max_iterations}) but couldn't summarize. Error: {str(e)}" finally: from agent import relay_llm @@ -2424,7 +2424,7 @@ def cleanup_task_resources(agent, task_id: str) -> None: _ra().cleanup_vm(task_id) except Exception as e: if agent.verbose_logging: - logger.warning(f"Failed to cleanup VM for task {task_id}: {e}") + logger.warning("Failed to cleanup VM for task %s: %s", task_id, e) try: headed = False try: @@ -2442,7 +2442,7 @@ def cleanup_task_resources(agent, task_id: str) -> None: _ra().cleanup_browser(task_id) except Exception as e: if agent.verbose_logging: - logger.warning(f"Failed to cleanup browser for task {task_id}: {e}") + logger.warning("Failed to cleanup browser for task %s: %s", task_id, e) def _build_partial_stream_stub( diff --git a/agent/context_compressor.py b/agent/context_compressor.py index e8688d253b2f..9e93ac23d444 100644 --- a/agent/context_compressor.py +++ b/agent/context_compressor.py @@ -25,7 +25,12 @@ import uuid from typing import Any, Dict, List, Optional -from agent.auxiliary_client import call_llm, _is_connection_error, aux_interrupt_protection +from agent.auxiliary_client import ( + AuxiliaryExplicitCancellation, + _is_connection_error, + aux_interrupt_protection, + call_llm, +) from agent.context_engine import ContextEngine, sanitize_memory_context from agent.error_classifier import FailoverReason, classify_api_error from agent.model_metadata import ( @@ -1828,6 +1833,11 @@ def get_active_compression_failure_cooldown( refresh: bool = False, ) -> Optional[Dict[str, Any]]: """Return the live compression-failure cooldown for the bound session.""" + if refresh: + # Transaction rollback must distinguish an authoritative empty row + # from a failed/unavailable durable read. The public return value + # cannot do so because it deliberately falls back to local state. + self._last_cooldown_refresh_was_authoritative = None now_mono = time.monotonic() local_state = None if self._summary_failure_cooldown_until > now_mono: @@ -1852,10 +1862,16 @@ def get_active_compression_failure_cooldown( try: state = getter(session_id) except sqlite3.Error as exc: + if refresh: + self._last_cooldown_refresh_was_authoritative = False logger.debug("compression failure cooldown lookup failed: %s", exc) return local_state except Exception: + if refresh: + self._last_cooldown_refresh_was_authoritative = False return local_state + if refresh: + self._last_cooldown_refresh_was_authoritative = True if not state: if refresh: if local_state is not None and self._cooldown_persist_failed: @@ -6056,6 +6072,7 @@ def compress( # — take the narrow rescan, miss a beyond-window fossil, and discard the # rehydrated state as cross-session leakage (#57835). _previous_summary_before_scan = self._previous_summary + _summary_has_user_turn_before_scan = getattr(self, "_summary_has_user_turn", None) # A persisted handoff summary can sit in the protected head after a # resume (commonly immediately after the system prompt). Search from # the first non-system message through the compression window. On the @@ -6264,11 +6281,19 @@ def _window_row(idx: int, msg: Dict[str, Any]): # Deriving the auto focus topic scans recent user turns — only pay # for it when a summary will actually be generated. summary_focus_topic = focus_topic or self._derive_auto_focus_topic(messages) - summary = self._generate_summary( - turns_to_summarize, - focus_topic=summary_focus_topic, - memory_context=memory_context, - ) + try: + summary = self._generate_summary( + turns_to_summarize, + focus_topic=summary_focus_topic, + memory_context=memory_context, + ) + except AuxiliaryExplicitCancellation: + # Explicit cancellation is a true no-op. Restore state mutated by + # the resume/handoff self-heal scan before the exception escapes to + # the outer transaction, which restores the transcript and lease. + self._previous_summary = _previous_summary_before_scan + self._summary_has_user_turn = _summary_has_user_turn_before_scan + raise # If summary generation failed, behavior splits on # ``abort_on_summary_failure`` (config: compression.abort_on_summary_failure): diff --git a/agent/conversation_compression.py b/agent/conversation_compression.py index 81b8ca71ac16..93a295e27cd8 100644 --- a/agent/conversation_compression.py +++ b/agent/conversation_compression.py @@ -42,6 +42,7 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Tuple +from agent.auxiliary_client import AuxiliaryExplicitCancellation from agent.context_engine import ( automatic_compaction_status_message, sanitize_memory_context, @@ -207,6 +208,202 @@ def _cached_prompt_reflects_builtin_memory(agent: Any, cached_prompt: str) -> bo return True +_COMPRESSOR_ATTEMPT_STATE_FIELDS = ( + "_previous_summary", + "_summary_has_user_turn", + "compression_count", + "_last_compression_savings_pct", + "_ineffective_compression_count", + "_anti_thrash_recovery_deadline", + "_fallback_compression_streak", + "_verify_compaction_cleared_threshold", + "_last_compression_made_progress", + "_summary_failure_cooldown_until", + "_cooldown_persist_failed", + "_last_summary_error", + "_consecutive_timeout_failures", + "_last_summary_dropped_count", + "_last_summary_fallback_used", + "_last_compress_aborted", + "_last_summary_auth_failure", + "_last_summary_network_failure", + "_last_aux_model_failure_error", + "_last_aux_model_failure_model", + "_summary_model_fallen_back", + "summary_model", + "_last_compression_telemetry", + "_active_compression_telemetry", + "_compression_telemetry_seed", +) + +_COMPRESSOR_COOLDOWN_STATE_FIELDS = ( + "_summary_failure_cooldown_until", + "_last_summary_error", + "_cooldown_persist_failed", +) + + +def _snapshot_compressor_attempt_state(compressor: Any) -> dict[str, Any]: + """Copy only mutable bookkeeping owned by one compression attempt. + + The explicit allow-list avoids copying provider clients, SessionDB handles, + locks, and plugin resources. Missing fields are intentionally ignored so + legacy and third-party compressors keep their existing contract. + """ + try: + values = vars(compressor) + except TypeError: + return {} + selected = { + name: values[name] + for name in _COMPRESSOR_ATTEMPT_STATE_FIELDS + if name in values + } + # Copy the collection as one object so aliases between fields (notably + # _active_compression_telemetry and _last_compression_telemetry) survive. + return copy.deepcopy(selected) + + +def _restore_compressor_attempt_state( + compressor: Any, + snapshot: dict[str, Any], + *, + durable_cooldown_authoritative: Optional[bool] = None, + durable_cooldown_state: Optional[dict[str, Any]] = None, +) -> None: + """Restore the safe per-attempt snapshot after a pre-commit hard cancel.""" + # A successful summary clears the durable cooldown before the outer commit + # boundary. Recreate (or clear) that row before restoring exact in-memory + # values, otherwise the next refresh would overwrite this rollback. Unknown + # durable state and intentionally unpersisted local cooldowns are never + # converted into destructive DB writes during cancellation. + if ( + "_summary_failure_cooldown_until" in snapshot + and durable_cooldown_authoritative is not False + and ( + durable_cooldown_authoritative is True + or not bool(snapshot.get("_cooldown_persist_failed", False)) + ) + ): + session_db = vars(compressor).get("_session_db") + session_id = vars(compressor).get("_session_id") + if session_db is not None and session_id: + if durable_cooldown_authoritative is True: + restorer = getattr( + type(session_db), + "restore_compression_failure_cooldown_row", + None, + ) + if not callable(restorer) or durable_cooldown_state is None: + raise RuntimeError( + "exact compression cooldown rollback API is unavailable" + ) + # This API restores raw columns (including expired and null + # combinations), verifies the read-back, and propagates failure. + restorer( + session_db, + session_id, + copy.deepcopy(durable_cooldown_state), + ) + else: + try: + deadline = float( + snapshot["_summary_failure_cooldown_until"] or 0.0 + ) + remaining = max(0.0, deadline - time.monotonic()) + durable_deadline = time.time() + remaining + durable_error = snapshot.get("_last_summary_error") + if remaining > 0: + recorder = getattr( + type(session_db), + "record_compression_failure_cooldown", + None, + ) + if callable(recorder): + recorder( + session_db, + session_id, + durable_deadline, + durable_error, + ) + else: + clearer = getattr( + type(session_db), + "clear_compression_failure_cooldown", + None, + ) + if callable(clearer): + clearer(session_db, session_id) + except Exception: + # Legacy/third-party compatibility path: its existing APIs + # do not provide a verifiable transaction contract. + logger.debug( + "compression cooldown persistence rollback failed", + exc_info=True, + ) + restored = copy.deepcopy(snapshot) + for name, value in restored.items(): + setattr(compressor, name, value) + + +def _capture_authoritative_cooldown_under_lease( + compressor: Any, + attempt_snapshot: dict[str, Any], +) -> tuple[Optional[bool], Optional[dict[str, Any]]]: + """Refresh and snapshot built-in durable cooldown state under the lease. + + Third-party compressors are deliberately not invoked here: arbitrary plugin + callbacks must not run while the session lease is held. A durable read + failure returns ``False`` so rollback cannot mistake unknown durable state + for an authoritative empty row and clear it; an unavailable legacy API + returns ``None`` and preserves the compatibility path. + """ + try: + from agent.context_compressor import ContextCompressor + + if not isinstance(compressor, ContextCompressor): + return None, None + values = vars(compressor) + session_db = values.get("_session_db") + session_id = values.get("_session_id") + raw_reader = ( + getattr( + type(session_db), "get_compression_failure_cooldown_row", None + ) + if session_db is not None + else None + ) + if session_db is None or not session_id: + # Unbound compressors have no durable row to mutate or restore. + return None, None + if not callable(raw_reader): + return False, None + # Capture the exact persisted representation first. The active getter + # intentionally filters expired rows and therefore cannot serve as a + # lossless rollback snapshot. + durable_state = raw_reader(session_db, session_id) + if not isinstance(durable_state, dict): + raise TypeError("raw compression cooldown snapshot must be a mapping") + ContextCompressor.get_active_compression_failure_cooldown( + compressor, + refresh=True, + ) + except Exception as exc: + logger.debug("authoritative compression cooldown capture failed: %s", exc) + return False, None + authoritative = getattr( + compressor, "_last_cooldown_refresh_was_authoritative", None + ) + if authoritative is not True: + return authoritative, None + + values = vars(compressor) + for name in _COMPRESSOR_COOLDOWN_STATE_FIELDS: + if name in values: + attempt_snapshot[name] = copy.deepcopy(values[name]) + return True, copy.deepcopy(durable_state) + + class CompressionCommitFence: """Fence timeout cancellation against post-summary session mutation. @@ -241,7 +438,7 @@ def seconds_since_progress(self) -> float: """Seconds since the worker last reported forward progress.""" return max(0.0, time.monotonic() - self._last_progress) - def cancel_before_commit(self) -> bool: + def cancel_before_commit(self, cancel_event: Any = None) -> bool: """Cancel a pending commit, or wait for an active commit to finish. Returns ``True`` when cancellation won before the commit boundary. @@ -250,8 +447,12 @@ def cancel_before_commit(self) -> bool: """ with self._lock: if self._commit_started: + if cancel_event is not None: + cancel_event.set() return False self._cancelled = True + if cancel_event is not None: + cancel_event.set() return True def try_cancel_before_commit(self) -> Optional[bool]: @@ -270,10 +471,13 @@ def try_cancel_before_commit(self) -> Optional[bool]: finally: self._lock.release() - def begin_commit(self) -> bool: - """Enter the commit boundary unless cancellation already won.""" + def begin_commit(self, cancel_event: Any = None) -> bool: + """Atomically admit commit unless a hard cancellation already won.""" self._lock.acquire() - if self._cancelled: + if self._cancelled or ( + cancel_event is not None and bool(cancel_event.is_set()) + ): + self._cancelled = True self._lock.release() return False self._commit_started = True @@ -307,13 +511,21 @@ def _lock_api_is_absent_on_session_db(lock_db: Any) -> bool: return False -def _refresh_persisted_compression_guards(compressor: Any) -> None: +def _refresh_persisted_compression_guards( + compressor: Any, + *, + include_cooldown: bool = True, +) -> None: """Refresh durable automatic-compression guards on a built-in compressor.""" - method_calls = ( - ("get_active_compression_failure_cooldown", {"refresh": True}), + method_calls = [ ("_load_fallback_compression_streak", {}), ("_load_ineffective_compression_count", {}), - ) + ] + if include_cooldown: + method_calls.insert( + 0, + ("get_active_compression_failure_cooldown", {"refresh": True}), + ) for method_name, kwargs in method_calls: method = getattr(type(compressor), method_name, None) if not callable(method): @@ -1298,6 +1510,11 @@ def compress_context( prompt — the session is NOT rotated. Callers should detect the no-op via ``len(returned) == len(input)`` and stop the retry loop. """ + _compressor_attempt_snapshot = _snapshot_compressor_attempt_state( + agent.context_compressor + ) + _durable_cooldown_authoritative: Optional[bool] = None + _durable_cooldown_state: Optional[dict[str, Any]] = None if ( defer_context_engine_notification and callable(getattr(agent, _PENDING_CONTEXT_ENGINE_NOTIFICATION, None)) @@ -1343,8 +1560,13 @@ def compress_context( if getattr(agent, "api_mode", None) == "codex_app_server": _codex_fence_entered = False if commit_fence is not None: - _codex_fence_entered = commit_fence.begin_commit() + _codex_fence_entered = commit_fence.begin_commit( + getattr(agent, "_hard_interrupt_requested", None) + ) if not _codex_fence_entered: + _restore_compressor_attempt_state( + agent.context_compressor, _compressor_attempt_snapshot + ) existing_prompt = getattr(agent, "_cached_system_prompt", None) if not existing_prompt: existing_prompt = agent._build_system_prompt(system_message) @@ -1671,13 +1893,36 @@ def _release_lock() -> None: ) return messages, _existing_sp + # Snapshot the authoritative durable cooldown only after this attempt owns + # the session lease. This runs for force=True too, but does not apply the + # automatic breaker gate: manual compression still retries immediately. + _durable_cooldown_authoritative, _durable_cooldown_state = ( + _capture_authoritative_cooldown_under_lease( + agent.context_compressor, + _compressor_attempt_snapshot, + ) + ) + if _durable_cooldown_authoritative is False: + # A bound built-in compressor reached its durable getter and the read + # failed. Proceeding with force=True could clear an unknown newer row + # before cancellation has enough information to restore it. This is a + # persistence-safety abort, not automatic breaker gating. + _release_lock() + existing_prompt = getattr(agent, "_cached_system_prompt", None) + if not existing_prompt: + existing_prompt = agent._build_system_prompt(system_message) + return messages, existing_prompt + # The agent may have been constructed before another path completed an # in-place compaction on the same session. Re-read durable breaker state # after acquiring the session lock so this final gate cannot act on the # stale snapshot loaded by bind_session_state(). if not force: compressor = agent.context_compressor - _refresh_persisted_compression_guards(compressor) + _refresh_persisted_compression_guards( + compressor, + include_cooldown=False, + ) blocked = getattr( type(compressor), "_automatic_compression_blocked", @@ -1691,6 +1936,7 @@ def _release_lock() -> None: return messages, existing_prompt _activity_heartbeat: Optional[_CompressionActivityHeartbeat] = None + messages_before_compression = None try: if _lock_holder is not None: _lock_refresher = _CompressionLockLeaseRefresher( @@ -1799,13 +2045,74 @@ def _release_lock() -> None: # provider that keeps the connection alive forever is cut off at the # streamed total ceiling (see _aux_stream_total_ceiling) instead of # outliving the SDK's inactivity timeout indefinitely. - from agent.auxiliary_client import aux_progress_hook + from agent.auxiliary_client import ( + aux_interrupt_protection, + aux_progress_hook, + ) _progress_hook = ( commit_fence.touch_progress if commit_fence is not None else (lambda: None) ) - with aux_progress_hook(_progress_hook): + # Incoming-message interrupts and active-turn redirects must not tear an + # atomic summary in half (#23975). Explicit stop surfaces set a separate + # Event atomically; never infer cause from the racy message fields. + _hard_cancel_event = getattr(agent, "_hard_interrupt_requested", None) + with aux_progress_hook(_progress_hook), aux_interrupt_protection( + cancel_event=_hard_cancel_event + ): compressed = compress_fn(messages, **compress_kwargs) + # Freeze a hard stop that arrived after the final provider attempt + # unwound but before this transaction can rotate session state. + if _hard_cancel_event is not None and _hard_cancel_event.is_set(): + raise AuxiliaryExplicitCancellation() + except AuxiliaryExplicitCancellation: + try: + _restore_compressor_attempt_state( + agent.context_compressor, + _compressor_attempt_snapshot, + durable_cooldown_authoritative=_durable_cooldown_authoritative, + durable_cooldown_state=_durable_cooldown_state, + ) + except BaseException as _rollback_exc: + # Compensation failure must surface, but it must not strand the + # session lease or retain an in-memory transcript mutation. + if ( + messages_before_compression is not None + and messages != messages_before_compression + ): + messages[:] = copy.deepcopy(messages_before_compression) + if _activity_heartbeat is not None: + _activity_heartbeat.stop("context compression rollback failed") + _activity_heartbeat = None + _release_lock() + _emit_compression_attempt_telemetry( + agent, + started_at=_attempt_started_at, + commit_status="aborted", + split_status="aborted", + failure_class=f"rollback:{type(_rollback_exc).__name__}", + ) + raise + if ( + messages_before_compression is not None + and messages != messages_before_compression + ): + messages[:] = copy.deepcopy(messages_before_compression) + if _activity_heartbeat is not None: + _activity_heartbeat.stop("context compression cancelled") + _activity_heartbeat = None + _release_lock() + _emit_compression_attempt_telemetry( + agent, + started_at=_attempt_started_at, + commit_status="aborted", + split_status="aborted", + failure_class="explicit_interrupt", + ) + _existing_sp = getattr(agent, "_cached_system_prompt", None) + if not _existing_sp: + _existing_sp = agent._build_system_prompt(system_message) + return messages, _existing_sp except BaseException as _compress_exc: # ANY exception after lock acquisition — memory hook, capability # inspection, engine lookup, or compress() — must release the lock so @@ -1918,8 +2225,19 @@ def _release_lock() -> None: return messages, _existing_sp if commit_fence is not None: - _commit_fence_entered = commit_fence.begin_commit() + _commit_fence_entered = commit_fence.begin_commit(_hard_cancel_event) if not _commit_fence_entered: + _restore_compressor_attempt_state( + agent.context_compressor, + _compressor_attempt_snapshot, + durable_cooldown_authoritative=_durable_cooldown_authoritative, + durable_cooldown_state=_durable_cooldown_state, + ) + if ( + messages_before_compression is not None + and messages != messages_before_compression + ): + messages[:] = copy.deepcopy(messages_before_compression) logger.info( "Compression commit cancelled before session mutation " "(session=%s).", diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index 68453df309f7..1884c06489d9 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -2672,7 +2672,7 @@ def _perform_api_call(next_api_kwargs): # Terminal — flush buffered retry trace so user sees what happened. agent._flush_status_buffer() agent._emit_status(f"❌ Max retries ({max_retries}) exceeded for invalid responses. Giving up.") - logger.error(f"{agent.log_prefix}Invalid API response after {max_retries} retries.") + logger.error("%sInvalid API response after %d retries.", agent.log_prefix, max_retries) agent._persist_session(messages, conversation_history) _final_response = f"Invalid API response after {max_retries} retries: {_failure_hint}" return { @@ -2687,7 +2687,7 @@ def _perform_api_call(next_api_kwargs): # Backoff before retry — jittered exponential: 5s base, 120s cap wait_time = jittered_backoff(retry_count, base_delay=5.0, max_delay=120.0) agent._buffer_vprint(f"⏳ Retrying in {wait_time:.1f}s ({_failure_hint})...") - logger.warning(f"Invalid API response (retry {retry_count}/{max_retries}): {', '.join(error_details)} | Provider: {provider_name}") + logger.warning("Invalid API response (retry %d/%d): %s | Provider: %s", retry_count, max_retries, ', '.join(error_details), provider_name) # Sleep in small increments to stay responsive to interrupts sleep_end = time.time() + wait_time @@ -4599,7 +4599,7 @@ def _perform_api_call(next_api_kwargs): agent._flush_status_buffer() agent._vprint(f"{agent.log_prefix}❌ Max compression attempts ({max_compression_attempts}) reached for payload-too-large error.", force=True) agent._vprint(f"{agent.log_prefix} 💡 Try /new to start a fresh conversation, or /compress to retry compression.", force=True) - logger.error(f"{agent.log_prefix}413 compression failed after {max_compression_attempts} attempts.") + logger.error("%s413 compression failed after %d attempts.", agent.log_prefix, max_compression_attempts) agent._persist_session(messages, conversation_history) _final_response = f"Request payload too large: max compression attempts ({max_compression_attempts}) reached." return { @@ -4668,7 +4668,7 @@ def _perform_api_call(next_api_kwargs): agent._flush_status_buffer() agent._vprint(f"{agent.log_prefix}❌ Payload too large and cannot compress further.", force=True) agent._vprint(f"{agent.log_prefix} 💡 Try /new to start a fresh conversation, or /compress to retry compression.", force=True) - logger.error(f"{agent.log_prefix}413 payload too large. Cannot compress further.") + logger.error("%s413 payload too large. Cannot compress further.", agent.log_prefix) agent._persist_session(messages, conversation_history) _final_response = "Request payload too large (413). Cannot compress further." return { @@ -4741,7 +4741,7 @@ def _perform_api_call(next_api_kwargs): agent._flush_status_buffer() agent._vprint(f"{agent.log_prefix}❌ Max compression attempts ({max_compression_attempts}) reached.", force=True) agent._vprint(f"{agent.log_prefix} 💡 Try /new to start a fresh conversation, or /compress to retry compression.", force=True) - logger.error(f"{agent.log_prefix}Context compression failed after {max_compression_attempts} attempts.") + logger.error("%sContext compression failed after %d attempts.", agent.log_prefix, max_compression_attempts) agent._persist_session(messages, conversation_history) _final_response = f"Context length exceeded: max compression attempts ({max_compression_attempts}) reached." return { @@ -4860,7 +4860,7 @@ def _perform_api_call(next_api_kwargs): agent._flush_status_buffer() agent._vprint(f"{agent.log_prefix}❌ Max compression attempts ({max_compression_attempts}) reached.", force=True) agent._vprint(f"{agent.log_prefix} 💡 Try /new to start a fresh conversation, or /compress to retry compression.", force=True) - logger.error(f"{agent.log_prefix}Context compression failed after {max_compression_attempts} attempts.") + logger.error("%sContext compression failed after %d attempts.", agent.log_prefix, max_compression_attempts) agent._persist_session(messages, conversation_history) _final_response = f"Context length exceeded: max compression attempts ({max_compression_attempts}) reached." return { @@ -4918,7 +4918,7 @@ def _perform_api_call(next_api_kwargs): agent._flush_status_buffer() agent._vprint(f"{agent.log_prefix}❌ Context length exceeded and cannot compress further.", force=True) agent._vprint(f"{agent.log_prefix} 💡 The conversation has accumulated too much content. Try /new to start fresh, or /compress to manually trigger compression.", force=True) - logger.error(f"{agent.log_prefix}Context length exceeded: {new_tokens:,} tokens. Cannot compress further.") + logger.error("%sContext length exceeded: %s tokens. Cannot compress further.", agent.log_prefix, f"{new_tokens:,}") agent._persist_session(messages, conversation_history) _final_response = f"Context length exceeded ({new_tokens:,} tokens). Cannot compress further." return { @@ -5184,7 +5184,7 @@ def _perform_api_call(next_api_kwargs): f"{agent.log_prefix} for localhost, or add the server's cert to your trust store.", force=True, ) - logger.error(f"{agent.log_prefix}Non-retryable client error: {api_error}") + logger.error("%sNon-retryable client error: %s", agent.log_prefix, api_error) # Skip session persistence when the error is likely # context-overflow related (status 400 + large session). # Persisting the failed user message would make the diff --git a/agent/interrupt_compat.py b/agent/interrupt_compat.py new file mode 100644 index 000000000000..bf56849495c7 --- /dev/null +++ b/agent/interrupt_compat.py @@ -0,0 +1,35 @@ +"""Compatibility helper for explicit agent stop producers.""" + +from __future__ import annotations + +import inspect +from typing import Any + + +def request_hard_interrupt(agent: Any, message: str | None = None) -> bool: + """Request an explicit stop, falling back to the legacy interrupt ABI. + + New agents expose ``hard_interrupt(message=None)``. Third-party agents and + old test doubles may only expose ``interrupt(message=None)``; keep those + usable without sending the newer ``hard_cancel=`` keyword they do not know. + Returns ``False`` only when neither callable is available. + """ + # Avoid treating a dynamic ``__getattr__`` proxy (notably an unspecced + # ``MagicMock`` or a third-party RPC facade) as if it genuinely implements + # the new ABI. Static lookup proves the attribute exists on the instance or + # its type before normal descriptor binding retrieves the callable. + try: + inspect.getattr_static(agent, "hard_interrupt") + except AttributeError: + interrupt = None + else: + interrupt = getattr(agent, "hard_interrupt", None) + if not callable(interrupt): + interrupt = getattr(agent, "interrupt", None) + if not callable(interrupt): + return False + if message is None: + interrupt() + else: + interrupt(message) + return True diff --git a/agent/model_metadata.py b/agent/model_metadata.py index 42d841009fa3..f89d25f4c8dd 100644 --- a/agent/model_metadata.py +++ b/agent/model_metadata.py @@ -1084,7 +1084,7 @@ def fetch_model_metadata(force_refresh: bool = False) -> Dict[str, Dict[str, Any return cache except Exception as e: - logger.warning(f"Failed to fetch model metadata from OpenRouter: {e}") + logger.warning("Failed to fetch model metadata from OpenRouter: %s", e) if _model_metadata_cache: return _model_metadata_cache disk_cache = _load_model_metadata_disk_cache() @@ -1185,8 +1185,22 @@ def fetch_endpoint_model_metadata( for candidate in candidates: url = candidate.rstrip("/") + "/models" + response = None try: - response = requests.get(url, headers=headers, timeout=(5, 10), verify=_resolve_requests_verify()) + response = requests.get( + url, + headers=headers, + timeout=(5, 10), + verify=_resolve_requests_verify(), + stream=True, + ) + if response.status_code in (401, 403): + logger.debug( + "Model metadata probe received HTTP %s from %s; stopping candidate probing", + response.status_code, + url, + ) + break response.raise_for_status() payload = response.json() cache: Dict[str, Dict[str, Any]] = {} @@ -1236,6 +1250,9 @@ def fetch_endpoint_model_metadata( return cache except Exception as exc: last_error = exc + finally: + if response is not None: + response.close() if last_error: logger.debug("Failed to fetch model metadata from %s/models: %s", normalized, last_error) diff --git a/agent/secret_scope.py b/agent/secret_scope.py index 8b376d5fceff..919fe3e27bd3 100644 --- a/agent/secret_scope.py +++ b/agent/secret_scope.py @@ -23,6 +23,7 @@ from __future__ import annotations import os +import re from contextvars import ContextVar, Token from pathlib import Path from typing import Dict, Mapping, Optional @@ -105,6 +106,14 @@ def current_secret_scope() -> Optional[Mapping[str, str]]: "VIRTUAL_ENV", "PYTHONPATH", "SSL_CERT_FILE", # Kanban paths (per-board, not per-profile-secret) "HERMES_KANBAN_DB", "HERMES_KANBAN_WORKSPACES_ROOT", "HERMES_KANBAN_BOARD", + # API-server LISTENER settings — deployment config (Docker compose + # ``environment:`` block, systemd ``Environment=``), not profile secrets. + # The scoped runner reload (#64674) must keep seeing them or container + # deployments silently lose the api_server platform (#69379). NOTE: + # API_SERVER_KEY is deliberately NOT here — it IS a credential and stays + # profile-scoped. + "API_SERVER_ENABLED", "API_SERVER_HOST", "API_SERVER_PORT", + "API_SERVER_CORS_ORIGINS", }) _GLOBAL_ENV_PREFIXES = ( "HERMES_KANBAN_", @@ -177,20 +186,72 @@ def get_secret(name: str, default: Optional[str] = None) -> Optional[str]: return val if val is not None else default +def _strip_inline_comment(value: str) -> str: + """Strip a dotenv-style inline comment from a raw ``.env`` value. + + Mirrors python-dotenv (1.2.2) semantics, verified empirically: + + - Quoted values: scan for the matching close quote + (backslash-escape-aware for double quotes, since ``save_env_value`` + writes ``\\"``/``\\\\`` escapes). Everything through the close quote is + kept; a trailing ``# ...`` remainder after it is discarded, so + ``KEY="has # inside" # trailing`` yields ``has # inside``. Non-comment + trailing junk leaves the value untouched (lenient, unlike dotenv's + hard parse error). + - Unquoted values: truncate only at a ``#`` PRECEDED BY WHITESPACE, so + ``KEY=foo#bar`` keeps ``foo#bar`` while ``KEY=value # comment`` keeps + ``value``. A value that *starts* with ``#`` (``KEY=#leading``) is kept. + """ + value = value.strip() + if not value: + return value + quote = value[0] + if quote in ("'", '"'): + i = 1 + while i < len(value): + ch = value[i] + if quote == '"' and ch == "\\": + i += 2 # skip the escaped character + continue + if ch == quote: + remainder = value[i + 1:].lstrip() + if remainder.startswith("#"): + return value[: i + 1] + return value + i += 1 + return value # unterminated quote: leave as-is + return re.split(r"\s+#", value, maxsplit=1)[0].strip() + + def load_env_file(env_path: Path) -> Dict[str, str]: """Parse a ``.env`` file into a plain dict WITHOUT touching ``os.environ``. Used to load a profile's secrets into an isolated mapping for - ``set_secret_scope``. Mirrors python-dotenv's basic parsing (KEY=VALUE, - ``export`` prefix, ``#`` comments, optional matching quotes) but never - mutates the process environment — that isolation is the whole point. + ``set_secret_scope``. Parses the small KEY=VALUE subset Hermes writes + itself (``export`` prefix, ``#`` comments — full-line and + dotenv-compatible inline, matching quotes with the + writer's ``\\"``/``\\\\`` escapes reversed — the same semantics as + ``hermes_cli.config._parse_env_value``) but never mutates the process + environment — that isolation is the whole point. + + Encoding is ``utf-8-sig`` so a leading UTF-8 BOM (Windows Notepad / + PowerShell ``Set-Content -Encoding UTF8``) does not prefix the first + key as ``\\ufeffNAME`` and make ``get_secret('NAME')`` miss under scope. """ secrets: Dict[str, str] = {} try: - text = env_path.read_text(encoding="utf-8") + text = env_path.read_text(encoding="utf-8-sig") except (FileNotFoundError, OSError, UnicodeDecodeError): return secrets + # Parse values with the canonical Hermes parser: save_env_value + # escapes " and \ inside double quotes, and every other reader + # (load_env, python-dotenv) reverses those escapes. Stripping only + # the outer quotes here would corrupt credentials containing " + # or \ — they work interactively but fail in scoped (cron / + # multiplex) resolution. + from hermes_cli.config import _parse_env_value + for raw in text.splitlines(): line = raw.strip() if not line or line.startswith("#"): @@ -203,10 +264,7 @@ def load_env_file(env_path: Path) -> Dict[str, str]: key = key.strip() if not key: continue - value = value.strip() - if len(value) >= 2 and value[0] == value[-1] and value[0] in ("'", '"'): - value = value[1:-1] - secrets[key] = value + secrets[key] = _parse_env_value(_strip_inline_comment(value)) return secrets diff --git a/agent/subagent_lifecycle.py b/agent/subagent_lifecycle.py index bbe63ad0930a..55e110aae5a6 100644 --- a/agent/subagent_lifecycle.py +++ b/agent/subagent_lifecycle.py @@ -21,6 +21,7 @@ from concurrent.futures import Future, TimeoutError from typing import Any, Callable, Mapping, Optional +from agent.interrupt_compat import request_hard_interrupt PUBLIC_CONTRACT_VERSION = 1 _MAX_GOAL_CHARS = 16_000 @@ -300,16 +301,22 @@ def cancel(self, handle: SubagentHandle, *, reason: str) -> SubagentCancelResult agent = record.agent record.state = SubagentState.CANCEL_REQUESTED record.updated_at = time.time() - if agent is None or not hasattr(agent, "interrupt"): + if agent is None: return SubagentCancelResult( False, unsupported=True, state=SubagentState.CANCEL_REQUESTED ) try: - agent.interrupt(f"Lifecycle cancellation requested: {reason[:500]}") + accepted = request_hard_interrupt( + agent, f"Lifecycle cancellation requested: {reason[:500]}" + ) except Exception: return SubagentCancelResult( False, unsupported=True, state=SubagentState.CANCEL_REQUESTED ) + if not accepted: + return SubagentCancelResult( + False, unsupported=True, state=SubagentState.CANCEL_REQUESTED + ) return SubagentCancelResult(True, state=SubagentState.CANCEL_REQUESTED) def result(self, handle: SubagentHandle) -> SubagentResult: diff --git a/agent/tool_executor.py b/agent/tool_executor.py index 6428d1ea19b8..c422c118e727 100644 --- a/agent/tool_executor.py +++ b/agent/tool_executor.py @@ -1191,8 +1191,8 @@ def _execute(next_args: dict[str, Any]) -> Any: logging.debug("file-mutation verifier record failed: %s", _ver_err) if agent.verbose_logging: - logging.debug(f"Tool {function_name} completed in {tool_duration:.2f}s") - logging.debug(f"Tool result ({len(function_result)} chars): {function_result}") + logging.debug("Tool %s completed in %.2fs", function_name, tool_duration) + logging.debug("Tool result (%d chars): %s", len(function_result), function_result) agent._current_tool = None _status_suffix = " (error)" if is_error else "" @@ -1251,7 +1251,7 @@ def _execute(next_args: dict[str, Any]) -> Any: result=display_function_result, ) except Exception as cb_err: - logging.debug(f"Tool progress callback error: {cb_err}") + logging.debug("Tool progress callback error: %s", cb_err) # Print cute message per tool if agent._should_emit_quiet_tool_messages(): @@ -1275,7 +1275,7 @@ def _execute(next_args: dict[str, Any]) -> Any: tc.id, name, display_args, display_function_result, ) except Exception as cb_err: - logging.debug(f"Tool complete callback error: {cb_err}") + logging.debug("Tool complete callback error: %s", cb_err) if ( risk_metadata is not None @@ -1899,9 +1899,9 @@ def _execute(next_args: dict) -> Any: agent._touch_activity(f"tool completed: {function_name} ({tool_duration:.1f}s){_status_suffix}") if agent.verbose_logging: - logging.debug(f"Tool {function_name} completed in {tool_duration:.2f}s") + logging.debug("Tool %s completed in %.2fs", function_name, tool_duration) _log_result = _multimodal_text_summary(function_result) - logging.debug(f"Tool result ({len(_log_result)} chars): {_log_result}") + logging.debug("Tool result (%d chars): %s", len(_log_result), _log_result) display_function_result = function_result function_result = maybe_persist_tool_result( @@ -1943,7 +1943,7 @@ def _execute(next_args: dict) -> Any: result=display_function_result, ) except Exception as cb_err: - logging.debug(f"Tool progress callback error: {cb_err}") + logging.debug("Tool progress callback error: %s", cb_err) if not _execution_blocked and agent.tool_complete_callback: try: @@ -1958,7 +1958,7 @@ def _execute(next_args: dict) -> Any: display_function_result, ) except Exception as cb_err: - logging.debug(f"Tool complete callback error: {cb_err}") + logging.debug("Tool complete callback error: %s", cb_err) if ( risk_metadata is not None diff --git a/cli.py b/cli.py index 559094a527d8..f62fdc603e95 100644 --- a/cli.py +++ b/cli.py @@ -54,6 +54,7 @@ from hermes_cli.cli_agent_setup_mixin import CLIAgentSetupMixin from hermes_cli.cli_commands_mixin import CLICommandsMixin from hermes_cli.cli_billing_mixin import CLIBillingMixin +from agent.interrupt_compat import request_hard_interrupt # prompt_toolkit for fixed input area TUI from prompt_toolkit.history import FileHistory @@ -15834,7 +15835,7 @@ def handle_ctrl_c(event): self._last_ctrl_c_time = now print("\n⚡ Interrupting agent... (press Ctrl+C again to force exit)") - self.agent.interrupt() + request_hard_interrupt(self.agent) # If there's text or images, clear them (like bash). # If everything is already empty, exit. elif event.app.current_buffer.text or self._attached_images: @@ -15912,7 +15913,7 @@ def handle_ctrl_q(event): if self._agent_running and self.agent: print("\n⚡ Interrupting agent...") - self.agent.interrupt() + request_hard_interrupt(self.agent) elif event.app.current_buffer.text or self._attached_images: event.app.current_buffer.reset() self._attached_images.clear() @@ -17492,8 +17493,11 @@ def _signal_handler(signum, frame): # minutes (#65998 class). Never raises. _arm_exit_watchdog_on_shutdown_signal() try: - if getattr(self, "agent", None) and getattr(self, "_agent_running", False): - self.agent.interrupt(f"received signal {signum}") + _signal_agent = getattr(self, "agent", None) + if _signal_agent is not None and getattr(self, "_agent_running", False): + request_hard_interrupt( + _signal_agent, f"received signal {signum}" + ) try: _grace = float(os.getenv("HERMES_SIGTERM_GRACE", "1.5")) except (TypeError, ValueError): @@ -17684,7 +17688,7 @@ def new_event_loop(self): # avoids wasted API calls and lets run_conversation clean up). if self.agent and getattr(self, '_agent_running', False): try: - self.agent.interrupt() + request_hard_interrupt(self.agent) except Exception: pass # Shut down voice recorder (release persistent audio stream) @@ -18105,7 +18109,7 @@ def _signal_handler_q(signum, frame): try: _agent = getattr(cli, "agent", None) if _agent is not None: - _agent.interrupt(f"received signal {signum}") + request_hard_interrupt(_agent, f"received signal {signum}") try: _grace = float(os.getenv("HERMES_SIGTERM_GRACE", "1.5")) except (TypeError, ValueError): diff --git a/contributors/emails/116476090+JeffStone69@users.noreply.github.com b/contributors/emails/116476090+JeffStone69@users.noreply.github.com new file mode 100644 index 000000000000..73f820cce9bd --- /dev/null +++ b/contributors/emails/116476090+JeffStone69@users.noreply.github.com @@ -0,0 +1 @@ +JeffStone69 diff --git a/contributors/emails/1762459322@qq.com b/contributors/emails/1762459322@qq.com new file mode 100644 index 000000000000..6decf7d09654 --- /dev/null +++ b/contributors/emails/1762459322@qq.com @@ -0,0 +1 @@ +ZachariahChu diff --git a/contributors/emails/286182457+Da7-Tech@users.noreply.github.com b/contributors/emails/286182457+Da7-Tech@users.noreply.github.com new file mode 100644 index 000000000000..b74f6d5e4275 --- /dev/null +++ b/contributors/emails/286182457+Da7-Tech@users.noreply.github.com @@ -0,0 +1,2 @@ +Da7-Tech +# PR #60420 salvage diff --git a/contributors/emails/jesse.casco@gmail.com b/contributors/emails/jesse.casco@gmail.com new file mode 100644 index 000000000000..c19672cddfde --- /dev/null +++ b/contributors/emails/jesse.casco@gmail.com @@ -0,0 +1 @@ +FixItFoundry diff --git a/contributors/emails/rkt.2@hotmail.com b/contributors/emails/rkt.2@hotmail.com new file mode 100644 index 000000000000..2819aa968450 --- /dev/null +++ b/contributors/emails/rkt.2@hotmail.com @@ -0,0 +1 @@ +sparkeros diff --git a/contributors/emails/shikanga-hermes@shikanga.co.uk b/contributors/emails/shikanga-hermes@shikanga.co.uk new file mode 100644 index 000000000000..1de07d04fa4d --- /dev/null +++ b/contributors/emails/shikanga-hermes@shikanga.co.uk @@ -0,0 +1,2 @@ +phantom-instruction-set +# PR #59076 salvage diff --git a/contributors/emails/suparious@users.noreply.github.com b/contributors/emails/suparious@users.noreply.github.com new file mode 100644 index 000000000000..994fe5b8902b --- /dev/null +++ b/contributors/emails/suparious@users.noreply.github.com @@ -0,0 +1 @@ +suparious diff --git a/contributors/emails/wayne1992127@gmail.com b/contributors/emails/wayne1992127@gmail.com new file mode 100644 index 000000000000..140d69963812 --- /dev/null +++ b/contributors/emails/wayne1992127@gmail.com @@ -0,0 +1 @@ +wayne1992127 diff --git a/cron/scheduler.py b/cron/scheduler.py index 327a048becc6..8cae9b69ca70 100644 --- a/cron/scheduler.py +++ b/cron/scheduler.py @@ -48,6 +48,7 @@ ) from hermes_cli.fallback_config import get_fallback_chain from hermes_time import now as _hermes_now +from agent.interrupt_compat import request_hard_interrupt logger = logging.getLogger(__name__) @@ -3632,8 +3633,7 @@ def _heartbeat_run_claim_if_due(): _last_desc, _iter_n, _iter_max, _cur_tool or "none", ) - if hasattr(agent, "interrupt"): - agent.interrupt("Cron job timed out (inactivity)") + request_hard_interrupt(agent, "Cron job timed out (inactivity)") raise TimeoutError( f"Cron job '{job_name}' idle for " f"{int(_secs_ago)}s (limit {int(_cron_inactivity_limit)}s) " diff --git a/docker/stage2-hook.sh b/docker/stage2-hook.sh index 05474ca927b4..899c8e86ac98 100755 --- a/docker/stage2-hook.sh +++ b/docker/stage2-hook.sh @@ -220,6 +220,11 @@ chown_hermes_tree() { echo "[stage2] Warning: chown $target failed (rootless container?) — continuing" } +tree_has_non_hermes_owner() { + target="$1" + find "$target" \( ! -user hermes -o ! -group hermes \) -print -quit 2>/dev/null | grep -q . +} + needs_chown=false if [ "$(stat -c %u "$HERMES_HOME" 2>/dev/null)" != "$actual_hermes_uid" ]; then needs_chown=true @@ -243,7 +248,7 @@ if [ "$needs_chown" = true ]; then # created and managed exclusively by hermes (see the s6-setuidgid mkdir # -p block below for the canonical list). for sub in cron sessions logs hooks memories skills skins plans workspace home profiles pairing platforms/pairing lazy-packages; do - if [ -e "$HERMES_HOME/$sub" ]; then + if [ -e "$HERMES_HOME/$sub" ] && tree_has_non_hermes_owner "$HERMES_HOME/$sub"; then chown_hermes_tree "$HERMES_HOME/$sub" fi done @@ -273,17 +278,20 @@ fi # are invoked via `docker exec hermes …` (which defaults # to root unless `-u` is passed), and that breaks the cont-init # reconciler (02-reconcile-profiles) which runs as hermes and walks -# the profiles dir. Idempotent; skipped on rootless containers where -# chown would fail. -if [ -d "$HERMES_HOME/profiles" ]; then +# the profiles dir. Skip the recursive walk when the tree is already +# owned correctly so warm boots do not rescan huge profile caches. +# Idempotent; skipped on rootless containers where chown would fail. +if [ -d "$HERMES_HOME/profiles" ] && tree_has_non_hermes_owner "$HERMES_HOME/profiles"; then chown_hermes_tree "$HERMES_HOME/profiles" fi # Always reset ownership of $HERMES_HOME/cron on every boot for the same # docker-exec/root-write reason as profiles/. The cron scheduler state # (jobs.json) must stay readable by the unprivileged hermes runtime even -# after root-context maintenance commands or scheduler writes. -if [ -d "$HERMES_HOME/cron" ]; then +# after root-context maintenance commands or scheduler writes. Skip the +# recursive walk when the tree is already owned correctly (same warm-boot +# gate as profiles/). +if [ -d "$HERMES_HOME/cron" ] && tree_has_non_hermes_owner "$HERMES_HOME/cron"; then chown_hermes_tree "$HERMES_HOME/cron" fi @@ -309,13 +317,14 @@ fi # silently leaving the approved user unauthorized (#10270). The targeted # data-volume chown above only runs when the top-level $HERMES_HOME is # mis-owned, so warm boots skip it — this block makes a container restart -# self-heal. Tiny directory (a handful of small JSON files), so the cost -# is negligible. -if [ -d "$HERMES_HOME/platforms/pairing" ]; then +# self-heal. Tiny directory (a handful of small JSON files), so even the +# ownership pre-scan is negligible; gated for consistency with profiles/ +# and cron/. +if [ -d "$HERMES_HOME/platforms/pairing" ] && tree_has_non_hermes_owner "$HERMES_HOME/platforms/pairing"; then chown_hermes_tree "$HERMES_HOME/platforms/pairing" fi # Legacy location (pre-consolidated layout). -if [ -d "$HERMES_HOME/pairing" ]; then +if [ -d "$HERMES_HOME/pairing" ] && tree_has_non_hermes_owner "$HERMES_HOME/pairing"; then chown_hermes_tree "$HERMES_HOME/pairing" fi diff --git a/gateway/authz_mixin.py b/gateway/authz_mixin.py index f1bb24bb8a85..144cb50431f4 100644 --- a/gateway/authz_mixin.py +++ b/gateway/authz_mixin.py @@ -456,7 +456,7 @@ def _is_user_authorized(self, source: SessionSource) -> bool: Platform.QQBOT: "QQ_GROUP_ALLOWED_USERS", }.get(source.platform, "") if chat_allowlist_env: - raw_chat_allowlist = os.getenv(chat_allowlist_env, "").strip() + raw_chat_allowlist = _platform_gate_env(chat_allowlist_env) if raw_chat_allowlist: allowed_group_ids = { cid.strip() @@ -498,7 +498,7 @@ def _is_user_authorized(self, source: SessionSource) -> bool: } if getattr(source, "is_bot", False): allow_bots_var = platform_allow_bots_map.get(source.platform) - if allow_bots_var and os.getenv(allow_bots_var, "none").lower().strip() in {"mentions", "all"}: + if allow_bots_var and _platform_gate_env(allow_bots_var, "none").lower().strip() in {"mentions", "all"}: return True if not user_id: @@ -876,13 +876,13 @@ def _get_unauthorized_dm_behavior( ), Platform.QQBOT: ("QQ_GROUP_ALLOWED_USERS",), } - if os.getenv(platform_env_map.get(platform, ""), "").strip(): + if _platform_gate_env(platform_env_map.get(platform, "")).strip(): return "ignore" for env_key in platform_group_env_map.get(platform, ()): - if os.getenv(env_key, "").strip(): + if _platform_gate_env(env_key).strip(): return "ignore" - if os.getenv("GATEWAY_ALLOWED_USERS", "").strip(): + if _platform_gate_env("GATEWAY_ALLOWED_USERS").strip(): return "ignore" return "pair" diff --git a/gateway/kanban_watchers.py b/gateway/kanban_watchers.py index 5fbbdea07b1e..dc2c5efd9011 100644 --- a/gateway/kanban_watchers.py +++ b/gateway/kanban_watchers.py @@ -1038,7 +1038,7 @@ async def _kanban_dispatcher_watcher(self) -> None: # Read max_spawn config to limit concurrent kanban tasks max_spawn = kanban_cfg.get("max_spawn", None) if max_spawn is not None: - logger.info(f"kanban dispatcher: max_spawn={max_spawn}") + logger.info("kanban dispatcher: max_spawn=%s", max_spawn) # Cap the number of simultaneously running tasks so slow workers # (local LLMs, resource-constrained hosts) don't pile up and time @@ -1063,7 +1063,7 @@ async def _kanban_dispatcher_watcher(self) -> None: ) max_in_progress = None else: - logger.info(f"kanban dispatcher: max_in_progress={max_in_progress}") + logger.info("kanban dispatcher: max_in_progress=%s", max_in_progress) raw_failure_limit = kanban_cfg.get("failure_limit", _kb.DEFAULT_FAILURE_LIMIT) try: diff --git a/gateway/pairing.py b/gateway/pairing.py index 4e4ec14f39c4..42ce7a89e582 100644 --- a/gateway/pairing.py +++ b/gateway/pairing.py @@ -146,6 +146,32 @@ def _user_ids_match(platform: str, left: str, right: str) -> bool: return bool(left_aliases and right_aliases and (left_aliases & right_aliases)) +def _read_allowlist_env(env_var: str) -> str: + """Read a platform allowlist env var through the profile secret scope. + + Under multiplexing the process env may hold ANOTHER profile's allowlist + (first-writer-wins YAML→env bridges), so reads must honor the installed + scope's verdict — including a scoped miss returning empty rather than + borrowing the process value. Unscoped callers (single-profile CLI / + admin endpoints) keep the legacy ``os.getenv`` read. + + TODO(profile-secrets): the grant mirror below still WRITES through + ``hermes_cli.config.save_env_value`` / ``remove_env_value``, which target + the root ``.env`` — those writes need a profile-aware counterpart before + pairing grants can be mirrored correctly under multiplexing. + """ + try: + from agent.secret_scope import UnscopedSecretError, get_secret + + try: + return (get_secret(env_var) or "").strip() + except UnscopedSecretError: + pass + except Exception: + pass + return (os.getenv(env_var) or "").strip() + + def _sync_allowlist_add(platform: str, user_id: str) -> None: """Add ``user_id`` to the platform allowlist env var IF one is configured. @@ -158,7 +184,7 @@ def _sync_allowlist_add(platform: str, user_id: str) -> None: env_var = _allowlist_env_for_platform(platform) if not env_var: return - current = os.getenv(env_var, "").strip() + current = _read_allowlist_env(env_var) if not current: return # No allowlist configured — leave the gateway open (option i). ids = _split_allowlist(current) @@ -278,7 +304,7 @@ def _sync_allowlist_remove(platform: str, user_id: str) -> None: env_var = _allowlist_env_for_platform(platform) if not env_var: return - current = os.getenv(env_var, "").strip() + current = _read_allowlist_env(env_var) if not current: return # No allowlist configured — do not touch config-only snapshots. ids = _split_allowlist(current) diff --git a/gateway/platforms/api_server.py b/gateway/platforms/api_server.py index f51d1d5834c8..d81aab5a9bd2 100644 --- a/gateway/platforms/api_server.py +++ b/gateway/platforms/api_server.py @@ -92,8 +92,33 @@ def _approval_event_choices(*, smart_denied: bool, allow_permanent: bool) -> lis validate_media_delivery_path, ) from agent.redact import redact_sensitive_text +from agent.interrupt_compat import request_hard_interrupt from gateway.readiness import collect_runtime_readiness +from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError +from agent.secret_scope import get_secret as _scoped_get_secret + + +def _get_scoped_secret(name, default=None): + """Scope-aware credential read with the default-profile startup fallback. + + Secondary profiles construct their adapters under a profile secret + scope -- the scope is authoritative and a scoped miss returns ``default`` + (no cross-profile borrow from ``os.environ``, which may hold another + profile's value). The DEFAULT profile's adapter constructs and sends + *unscoped* under multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash this path; there ``os.environ`` is that + profile's own value, so fall back to it. Same pattern as the Slack + ``SLACK_APP_TOKEN`` read (#59739) and + ``gateway/platforms/whatsapp_common.py::_get_wsecret``. + """ + try: + val = _scoped_get_secret(name, default) + except _UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + logger = logging.getLogger(__name__) @@ -1300,7 +1325,7 @@ def __init__(self, config: PlatformConfig): if raw_port is None: raw_port = os.getenv("API_SERVER_PORT", str(DEFAULT_PORT)) self._port: int = _coerce_port(raw_port, DEFAULT_PORT) - self._api_key: str = extra.get("key", os.getenv("API_SERVER_KEY", "")) + self._api_key: str = extra.get("key", _get_scoped_secret("API_SERVER_KEY", "")) self._cors_origins: tuple[str, ...] = self._parse_cors_origins( extra.get("cors_origins", os.getenv("API_SERVER_CORS_ORIGINS", "")), ) @@ -4335,7 +4360,7 @@ async def _emit(item): agent = agent_ref[0] if agent_ref else None if agent is not None: try: - agent.interrupt("SSE client disconnected") + request_hard_interrupt(agent, "SSE client disconnected") except Exception: pass _reap_disconnected_agent_processes(agent) @@ -4915,7 +4940,7 @@ async def _flush_batch() -> None: agent = agent_ref[0] if agent_ref else None if agent is not None: try: - agent.interrupt("SSE client disconnected") + request_hard_interrupt(agent, "SSE client disconnected") except Exception: pass _reap_disconnected_agent_processes(agent) @@ -4935,7 +4960,7 @@ async def _flush_batch() -> None: agent = agent_ref[0] if agent_ref else None if agent is not None: try: - agent.interrupt("SSE task cancelled") + request_hard_interrupt(agent, "SSE task cancelled") except Exception: pass # Same abandonment as a client disconnect: the run will never @@ -5590,12 +5615,29 @@ async def _handle_cron_fire(self, request: "web.Request") -> "web.Response": token = auth[7:].strip() if auth.startswith("Bearer ") else "" cfg = load_config() - claims = get_fire_verifier()( + verifier = get_fire_verifier() + verify_kwargs = dict( token=token, expected_audience=cfg_get(cfg, "cron", "chronos", "expected_audience", default=""), jwks_or_key=cfg_get(cfg, "cron", "chronos", "nas_jwks_url", default="") or None, issuer=cfg_get(cfg, "cron", "chronos", "portal_url", default="") or None, ) + try: + if asyncio.iscoroutinefunction(verifier): + claims = await verifier(**verify_kwargs) + else: + # The verifier resolves the NAS signing key from a JWKS URL, + # which is a synchronous HTTP GET on a cache miss (cold client + # or a rotated kid) — keep that blocking I/O off the event loop + # so a slow or rate-limited portal can't stall every other + # adapter sharing this loop. Same hardening the platform HTTP + # event verifier already got. + claims = await asyncio.to_thread(verifier, **verify_kwargs) + except Exception: + # Fail closed: a crashing verifier must never admit a fire — this + # is the only inbound that can trigger remote job execution. + logger.exception("cron fire: verifier crashed; rejecting token") + claims = None if claims is None: logger.warning( "cron fire: rejected invalid token: %s", @@ -6788,7 +6830,7 @@ async def _handle_stop_run(self, request: "web.Request") -> "web.Response": if agent is not None: try: - agent.interrupt("Stop requested via API") + request_hard_interrupt(agent, "Stop requested via API") except Exception: pass # The stopped run is abandoned — reap only the background diff --git a/gateway/platforms/base.py b/gateway/platforms/base.py index d7654ff6c147..de60322718bb 100644 --- a/gateway/platforms/base.py +++ b/gateway/platforms/base.py @@ -5545,11 +5545,16 @@ async def handle_message(self, event: MessageEvent) -> None: coerce_plaintext_gateway_command(event) - # Rewrite ``event.source.thread_id`` via the installed recovery hook - # (Telegram DM topic mode) so the session key, guard checks, and - # downstream delivery all agree on the same lane. - # Offloaded: the sync hook must not block the loop. - await asyncio.to_thread(self._apply_topic_recovery, event) + # Telegram topic recovery only applies to private DM topic lanes. Do + # not submit a no-op check for group/forum/channel traffic to the + # shared default executor: a busy pool would delay message dispatch. + needs_topic_recovery = ( + getattr(self, "_topic_recovery_fn", None) is not None + and event.source.platform == Platform.TELEGRAM + and event.source.chat_type == "dm" + ) + if needs_topic_recovery: + await asyncio.to_thread(self._apply_topic_recovery, event) session_key = build_session_key( event.source, diff --git a/gateway/platforms/bluebubbles.py b/gateway/platforms/bluebubbles.py index 1cb8c040cd90..ecc6baf45e01 100644 --- a/gateway/platforms/bluebubbles.py +++ b/gateway/platforms/bluebubbles.py @@ -56,6 +56,30 @@ "audio/aac": ".m4a", # preserves historical bluebubbles mapping (shared table says .aac) } +from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError +from agent.secret_scope import get_secret as _scoped_get_secret + + +def _get_scoped_secret(name, default=None): + """Scope-aware credential read with the default-profile startup fallback. + + Secondary profiles construct their adapters under a profile secret + scope -- the scope is authoritative and a scoped miss returns ``default`` + (no cross-profile borrow from ``os.environ``, which may hold another + profile's value). The DEFAULT profile's adapter constructs and sends + *unscoped* under multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash this path; there ``os.environ`` is that + profile's own value, so fall back to it. Same pattern as the Slack + ``SLACK_APP_TOKEN`` read (#59739) and + ``gateway/platforms/whatsapp_common.py::_get_wsecret``. + """ + try: + val = _scoped_get_secret(name, default) + except _UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- @@ -148,7 +172,7 @@ def __init__(self, config: PlatformConfig): self.server_url = _normalize_server_url( extra.get("server_url") or os.getenv("BLUEBUBBLES_SERVER_URL", "") ) - self.password = extra.get("password") or os.getenv("BLUEBUBBLES_PASSWORD", "") + self.password = extra.get("password") or _get_scoped_secret("BLUEBUBBLES_PASSWORD", "") self.webhook_host = ( extra.get("webhook_host") or os.getenv("BLUEBUBBLES_WEBHOOK_HOST", DEFAULT_WEBHOOK_HOST) diff --git a/gateway/platforms/qqbot/adapter.py b/gateway/platforms/qqbot/adapter.py index 8f5d7fececf9..d540692c095f 100644 --- a/gateway/platforms/qqbot/adapter.py +++ b/gateway/platforms/qqbot/adapter.py @@ -147,6 +147,31 @@ def _coerce_list(value: Any) -> List[str]: return _coerce_list_impl(value) +def _resolve_qq_secret(name: str, default: str = "") -> str: + """Resolve a per-profile ``QQ_*`` setting honoring the active secret scope. + + When a profile secret scope is installed — every secondary multiplex + profile is constructed and handled inside ``_profile_runtime_scope`` + (``gateway/run.py``), as is each per-turn inbound message — read from it so + profiles never see each other's ``os.environ`` values. This is the + cross-profile credential collision fixed for the WeChat adapter in #59662. + + The primary/active profile is constructed without a scope and legitimately + owns ``os.environ``, so fall back to it there instead of failing closed: a + bare ``get_secret`` would raise ``UnscopedSecretError`` on the active + profile's ``__init__`` and break its startup. Same pattern as the Slack + ``SLACK_APP_TOKEN`` read (#59739) and + ``gateway.platforms.whatsapp_common._get_wsecret``. + """ + from agent.secret_scope import UnscopedSecretError, get_secret + + try: + val = get_secret(name, default) + except UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + # --------------------------------------------------------------------------- # QQAdapter # --------------------------------------------------------------------------- @@ -202,9 +227,11 @@ def __init__(self, config: PlatformConfig): super().__init__(config, Platform.QQBOT) extra = config.extra or {} - self._app_id = str(extra.get("app_id") or os.getenv("QQ_APP_ID", "")).strip() + self._app_id = str( + extra.get("app_id") or _resolve_qq_secret("QQ_APP_ID", "") + ).strip() self._client_secret = str( - extra.get("client_secret") or os.getenv("QQ_CLIENT_SECRET", "") + extra.get("client_secret") or _resolve_qq_secret("QQ_CLIENT_SECRET", "") ).strip() self._markdown_support = bool(extra.get("markdown_support", True)) @@ -2202,13 +2229,13 @@ def _resolve_stt_config(self) -> Optional[Dict[str, str]]: } # 2. QQ-specific env vars (set by `hermes setup gateway` / `hermes gateway`) - qq_stt_key = os.getenv("QQ_STT_API_KEY", "") + qq_stt_key = _resolve_qq_secret("QQ_STT_API_KEY", "") if qq_stt_key: - base_url = os.getenv( + base_url = _resolve_qq_secret( "QQ_STT_BASE_URL", "https://open.bigmodel.cn/api/coding/paas/v4", ) - model = os.getenv("QQ_STT_MODEL", "glm-asr") + model = _resolve_qq_secret("QQ_STT_MODEL", "glm-asr") return { "base_url": base_url.rstrip("/"), "api_key": qq_stt_key, @@ -3170,7 +3197,7 @@ def _strip_at_mention(content: str) -> str: def _open_dm_opted_in(self) -> bool: if os.getenv("GATEWAY_ALLOW_ALL_USERS", "").lower() in {"true", "1", "yes"}: return True - return os.getenv("QQ_ALLOW_ALL_USERS", "").lower() in {"true", "1", "yes"} + return _resolve_qq_secret("QQ_ALLOW_ALL_USERS", "").lower() in {"true", "1", "yes"} def _is_dm_allowed(self, user_id: str) -> bool: if self._dm_policy == "disabled": diff --git a/gateway/platforms/weixin.py b/gateway/platforms/weixin.py index b44ce1ee698f..68a0e0a9e1b1 100644 --- a/gateway/platforms/weixin.py +++ b/gateway/platforms/weixin.py @@ -68,7 +68,25 @@ ) from hermes_constants import get_hermes_home from utils import atomic_json_write -from agent.secret_scope import get_secret +from agent.secret_scope import UnscopedSecretError, get_secret + + +def _wx_secret(name: str, default: Optional[str] = None) -> Optional[str]: + """Scope-aware WEIXIN_* read with the default-profile startup fallback. + + Secondary profiles construct their adapters under + ``_profile_runtime_scope`` — the scope is authoritative and a scoped miss + returns ``default`` (no cross-profile borrow from ``os.environ``). The + DEFAULT profile's adapter constructs and sends *unscoped* under + multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash its Weixin path; there ``os.environ`` is + that profile's own value, so fall back to it. Same pattern as the Slack + ``SLACK_APP_TOKEN`` read (#59739) and WhatsApp's ``_get_wsecret``. + """ + try: + return get_secret(name, default) + except UnscopedSecretError: + return os.getenv(name, default) ILINK_BASE_URL = "https://ilinkai.weixin.qq.com" WEIXIN_CDN_BASE_URL = "https://novac2c.cdn.weixin.qq.com/c2c" @@ -1175,11 +1193,11 @@ def __init__(self, config: PlatformConfig): self._poll_task: Optional[asyncio.Task] = None self._dedup = MessageDeduplicator(ttl_seconds=MESSAGE_DEDUP_TTL_SECONDS) - self._account_id = str(extra.get("account_id") or get_secret("WEIXIN_ACCOUNT_ID", "")).strip() - self._token = str(config.token or extra.get("token") or get_secret("WEIXIN_TOKEN", "")).strip() - self._base_url = str(extra.get("base_url") or get_secret("WEIXIN_BASE_URL", ILINK_BASE_URL)).strip().rstrip("/") + self._account_id = str(extra.get("account_id") or _wx_secret("WEIXIN_ACCOUNT_ID", "")).strip() + self._token = str(config.token or extra.get("token") or _wx_secret("WEIXIN_TOKEN", "")).strip() + self._base_url = str(extra.get("base_url") or _wx_secret("WEIXIN_BASE_URL", ILINK_BASE_URL)).strip().rstrip("/") self._cdn_base_url = str( - extra.get("cdn_base_url") or get_secret("WEIXIN_CDN_BASE_URL", WEIXIN_CDN_BASE_URL) + extra.get("cdn_base_url") or _wx_secret("WEIXIN_CDN_BASE_URL", WEIXIN_CDN_BASE_URL) ).strip().rstrip("/") self._send_chunk_delay_seconds = float( extra.get("send_chunk_delay_seconds") or os.getenv("WEIXIN_SEND_CHUNK_DELAY_SECONDS", "1.5") @@ -2313,10 +2331,10 @@ async def send_weixin_direct( This bypasses the long-poll adapter lifecycle and uses the raw API directly. """ - account_id = str(extra.get("account_id") or get_secret("WEIXIN_ACCOUNT_ID", "")).strip() - base_url = str(extra.get("base_url") or get_secret("WEIXIN_BASE_URL", ILINK_BASE_URL)).strip().rstrip("/") - cdn_base_url = str(extra.get("cdn_base_url") or get_secret("WEIXIN_CDN_BASE_URL", WEIXIN_CDN_BASE_URL)).strip().rstrip("/") - resolved_token = str(token or extra.get("token") or get_secret("WEIXIN_TOKEN", "")).strip() + account_id = str(extra.get("account_id") or _wx_secret("WEIXIN_ACCOUNT_ID", "")).strip() + base_url = str(extra.get("base_url") or _wx_secret("WEIXIN_BASE_URL", ILINK_BASE_URL)).strip().rstrip("/") + cdn_base_url = str(extra.get("cdn_base_url") or _wx_secret("WEIXIN_CDN_BASE_URL", WEIXIN_CDN_BASE_URL)).strip().rstrip("/") + resolved_token = str(token or extra.get("token") or _wx_secret("WEIXIN_TOKEN", "")).strip() if not resolved_token: return {"error": "Weixin token missing. Configure WEIXIN_TOKEN or platforms.weixin.token."} if not account_id: diff --git a/gateway/run.py b/gateway/run.py index 532ff9c2a942..e184be76fb99 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -59,6 +59,7 @@ ) from agent.conversation_loop import INTERRUPT_WAITING_FOR_MODEL_PREFIX from agent.i18n import t +from agent.interrupt_compat import request_hard_interrupt from hermes_cli.config import cfg_get from hermes_cli.fallback_config import get_fallback_chain @@ -2204,6 +2205,7 @@ def _platform_has_bot_credential(platform: "Platform", platform_config: "Platfor _BUILTIN_PLATFORM_VALUES, GatewayConfig, PlatformConfig, + _getenv, load_gateway_config, ) from gateway.session import ( @@ -2293,11 +2295,11 @@ def _own_policy_open_startup_violation(config) -> Optional[str]: extra = getattr(platform_config, "extra", None) or {} dm_policy = str( extra.get("dm_policy") - or (os.getenv(dm_env, "pairing") if dm_env else "pairing") + or (_getenv(dm_env, "pairing") if dm_env else "pairing") ).strip().lower() group_policy = str( extra.get("group_policy") - or (os.getenv(group_env, "pairing") if group_env else "pairing") + or (_getenv(group_env, "pairing") if group_env else "pairing") ).strip().lower() if dm_policy != "open" and group_policy != "open": continue @@ -2306,7 +2308,7 @@ def _own_policy_open_startup_violation(config) -> Optional[str]: ).lower() in {"true", "1", "yes"} platform_opted_in = gateway_allow_all or ( allow_all_env - and os.getenv(allow_all_env, "").lower() in {"true", "1", "yes"} + and _getenv(allow_all_env, "").lower() in {"true", "1", "yes"} ) if platform_opted_in: continue @@ -2779,9 +2781,9 @@ def _abandon_timed_out_gateway_turn( timeout_fired.set() agent = agent_holder[0] if agent_holder else None - if agent is not None and hasattr(agent, "interrupt"): + if agent is not None: try: - agent.interrupt(_INTERRUPT_REASON_TIMEOUT) + request_hard_interrupt(agent, _INTERRUPT_REASON_TIMEOUT) except Exception: logger.debug("Timed-out agent interrupt failed", exc_info=True) @@ -9044,7 +9046,7 @@ def _interrupt_running_agents(self, reason: str) -> None: if agent is _AGENT_PENDING_SENTINEL: continue try: - agent.interrupt(reason) + request_hard_interrupt(agent, reason) logger.debug("Interrupted running agent for session %s during shutdown", session_key) except Exception as e: logger.debug("Failed interrupting agent during shutdown: %s", e) @@ -22289,7 +22291,7 @@ async def _interrupt_and_clear_session( _process_task_id = "" _process_baseline = None if running_agent and running_agent is not _AGENT_PENDING_SENTINEL: - running_agent.interrupt(interrupt_reason) + request_hard_interrupt(running_agent, interrupt_reason) _process_task_id = getattr( running_agent, "_gateway_turn_process_task_id", "" ) @@ -23072,7 +23074,18 @@ async def _run_agent_via_proxy( "tools": [], } - proxy_key = os.getenv("GATEWAY_PROXY_KEY", "").strip() + # Scope-aware read: the proxy key is a per-profile credential; under + # multiplex honor the installed scope's verdict (Slack pattern for + # the unscoped default-profile loop). + try: + from agent.secret_scope import UnscopedSecretError, get_secret + + try: + proxy_key = (get_secret("GATEWAY_PROXY_KEY") or "").strip() + except UnscopedSecretError: + proxy_key = os.getenv("GATEWAY_PROXY_KEY", "").strip() + except Exception: + proxy_key = os.getenv("GATEWAY_PROXY_KEY", "").strip() def _run_still_current() -> bool: if run_generation is None or not session_key: @@ -24529,8 +24542,8 @@ def _run_sync_with_timeout_lifecycle(): # Interrupt the agent if it's still running so the thread # pool worker is freed. - if _timed_out_agent and hasattr(_timed_out_agent, "interrupt"): - _timed_out_agent.interrupt(_INTERRUPT_REASON_TIMEOUT) + if _timed_out_agent: + request_hard_interrupt(_timed_out_agent, _INTERRUPT_REASON_TIMEOUT) _timeout_mins = int(_agent_timeout // 60) or 1 @@ -25305,6 +25318,7 @@ def _start_gateway_housekeeping(stop_event: threading.Event, adapters=None, loop PASTE_SWEEP_EVERY = 60 # ticks — once per hour CURATOR_EVERY = 60 # ticks — poll hourly (inner gate handles the real cadence) AUTO_ARCHIVE_EVERY = 60 # ticks — poll hourly (state_meta gate owns the real cadence) + MEMORY_TRIM_EVERY = 1 # shared helper cooldown bounds actual allocator work # Every platform media cache prunes on the same hourly cadence — one loop # over (name, cleanup_fn), not a copy-pasted try/except per cache. @@ -25413,6 +25427,25 @@ def _start_gateway_housekeeping(stop_event: threading.Event, adapters=None, loop except Exception as e: logger.debug("Auto-archive tick error: %s", e) + # This is the long-lived messaging-gateway counterpart to the TUI idle + # reaper. The helper is config-gated and rate-limited, so calling it on + # the 60s housekeeping cadence does not create a trim storm. + if tick_count % MEMORY_TRIM_EVERY == 0: + try: + from hermes_cli.mem_trim import trim_memory + + trim_memory(reason="messaging gateway housekeeping") + except Exception as exc: + # debug, not warning: sibling housekeeping branches all log + # failures at debug, and a persistent failure (e.g. broken + # import after a partial update) would otherwise warn every + # 60s forever. + logger.debug( + "gateway housekeeping memory trim failed: %s: %s", + type(exc).__name__, + exc, + ) + stop_event.wait(timeout=interval) logger.info("Gateway housekeeping stopped") diff --git a/gateway/session.py b/gateway/session.py index 2d5cebd32026..92e2a5ae099d 100644 --- a/gateway/session.py +++ b/gateway/session.py @@ -387,7 +387,19 @@ def _slack_tools_loaded() -> bool: except Exception: pass - if not (os.environ.get("SLACK_BOT_TOKEN") or "").strip(): + # Presence check through the profile secret scope: under multiplex the + # process env may carry another profile's token (Slack pattern for the + # unscoped default-profile path). + try: + from agent.secret_scope import UnscopedSecretError, get_secret + + try: + _slack_token = get_secret("SLACK_BOT_TOKEN") or "" + except UnscopedSecretError: + _slack_token = os.environ.get("SLACK_BOT_TOKEN") or "" + except Exception: + _slack_token = os.environ.get("SLACK_BOT_TOKEN") or "" + if not _slack_token.strip(): return False try: from hermes_cli.config import load_config @@ -1212,6 +1224,11 @@ def __init__(self, sessions_dir: Path, config: GatewayConfig, self._save_lock = threading.Lock() self._routing_generation = 0 self._persisted_routing_generation = 0 + # Single-entry upserts persisted since the last full rewrite: + # session_key -> (revision, entry_json). Revisions are allocated + # from _routing_generation, so fast and full snapshots are totally + # ordered; guarded by _save_lock (see _save_entry). + self._fast_persisted_entries: Dict[str, tuple[int, str]] = {} self._inflight_lock = threading.Lock() self._inflight_sessions: Dict[str, _SessionFlight] = {} # An unscoped pre-migration Slack key can represent at most one @@ -1459,12 +1476,23 @@ def _save(self) -> None: data, generation = self._snapshot_routing_locked() self._persist_routing_data(data, generation) + def _next_routing_generation_locked(self) -> int: + """Bump and return the shared routing counter. Caller holds ``_lock``. + + BOTH full snapshots (_snapshot_routing_locked) and single-entry fast + saves (_save_entry) MUST allocate from this one counter — the stale- + write protection in _persist_routing_data/_save_entry is a total order + over serialization times and silently breaks if the two paths ever + number themselves independently. + """ + self._routing_generation = getattr(self, "_routing_generation", 0) + 1 + return self._routing_generation + def _snapshot_routing_locked(self) -> tuple[Dict[str, Any], int]: """Capture immutable routing data and a monotonic generation.""" - self._routing_generation = getattr(self, "_routing_generation", 0) + 1 return ( {key: entry.to_dict() for key, entry in self._entries.items()}, - self._routing_generation, + self._next_routing_generation_locked(), ) def _persist_routing_data(self, data: Dict[str, Any], generation: int) -> None: @@ -1476,6 +1504,16 @@ def _persist_routing_data(self, data: Dict[str, Any], generation: int) -> None: with save_lock: if generation <= getattr(self, "_persisted_routing_generation", 0): return + # Fold in single-entry upserts with a newer revision than this + # snapshot (see _save_entry): revisions share the routing + # generation counter, so a fast record numbered above us was + # serialized after us and a delayed full rewrite must not + # regress it. + fast_persisted = getattr(self, "_fast_persisted_entries", None) + if fast_persisted: + for key, (revision, entry_json) in fast_persisted.items(): + if revision > generation: + data[key] = json.loads(entry_json) db_saved = False _db = getattr(self, "_db", None) if _db: @@ -1494,6 +1532,14 @@ def _persist_routing_data(self, data: Dict[str, Any], generation: int) -> None: if getattr(self, "_write_sessions_json", True) or not db_saved: self._save_sessions_json(data) self._persisted_routing_generation = generation + # This rewrite supersedes fast records at or below its + # generation; newer ones stay for the next delayed full writer. + if fast_persisted: + for key in [ + k for k, (rev, _) in fast_persisted.items() + if rev <= generation + ]: + del fast_persisted[key] def _save_sessions_json(self, data: Dict[str, Any]) -> None: """Write the legacy sessions.json mirror of the routing index.""" @@ -1540,6 +1586,84 @@ def _save_entries(self) -> None: with self._lock: data, generation = self._snapshot_routing_locked() self._persist_routing_data(data, generation) + + def _save_entry(self, session_key: str) -> None: + """Persist ONE routing entry via UPSERT — the per-turn fast path. + + The steady-state turn only bumps ``updated_at`` / + ``last_prompt_tokens`` on one entry; routing that through the + full index rewrite re-serializes every entry, DELETE+INSERTs + every gateway_routing row, and dumps+fsyncs a multi-MB + sessions.json — ~50ms p50 at ~1100 routing keys, and it runs + twice per turn. A single-row UPSERT keeps the durable state.db + mapping current in well under a millisecond. + + Correctness constraints this path relies on: + + - The key -> session_id mapping never changes here. Structural + transitions (create/recover/reset/switch/prune, and + compression-tip heals — see get_or_create_session) still use + the full-rewrite path, which also refreshes the legacy + sessions.json mirror. Between structural saves the mirror may + lag in metadata only; every remaining sessions.json reader is + a legacy fallback and state.db stays primary, so restart + rebinding is unaffected. + + - Ordering vs concurrent writers: the entry is serialized under + ``_lock`` together with a revision allocated from the routing + generation counter, so every snapshot — fast or full — carries + a unique, monotonically increasing number, and a higher number + always means same-or-newer data for this key. Under + ``_save_lock`` the upsert is skipped when a snapshot numbered + above ours already persisted this key: a FULL snapshot + (``_persisted_routing_generation``) or another fast save of + the same key (``_fast_persisted_entries``). Either contains a + same-or-newer copy, so writing ours would regress it. The + reverse interaction — a delayed full rewrite landing after a + later-serialized fast save — is handled in + ``_persist_routing_data``, which folds fast records numbered + above its snapshot into the rewrite. An older snapshot can + therefore never overwrite a newer one, in either direction. + + - No DB, or a failed upsert, falls back to the full rewrite so + DB-less installs keep sessions.json — their primary store — + durable every turn. + """ + with self._lock: + entry = self._entries.get(session_key) + if entry is None: + return + entry_json = json.dumps(entry.to_dict()) + revision = self._next_routing_generation_locked() + _db = getattr(self, "_db", None) + saver = getattr(_db, "save_gateway_routing_entry", None) if _db else None + if callable(saver): + save_lock = getattr(self, "_save_lock", None) + if save_lock is None: + save_lock = threading.Lock() + self._save_lock = save_lock + try: + with save_lock: + if getattr(self, "_persisted_routing_generation", 0) >= revision: + return + fast_persisted = getattr(self, "_fast_persisted_entries", None) + if fast_persisted is None: + fast_persisted = {} + self._fast_persisted_entries = fast_persisted + persisted = fast_persisted.get(session_key) + if persisted is not None and persisted[0] >= revision: + return + saver(session_key, entry_json, scope=self._routing_scope()) + fast_persisted[session_key] = (revision, entry_json) + return + except Exception as exc: + logger.warning( + "gateway.session: single-entry routing save failed for %r " + "(%s); falling back to full index rewrite", + session_key, exc, + ) + self._save_entries() + def _resolve_profile_for_key(self, source: Optional[SessionSource] = None) -> Optional[str]: """Return the profile namespace for session keys, or None when off. @@ -2334,6 +2458,11 @@ def _get_or_create_session_impl( # ---- Phase 2: lock write -- apply decisions to _entries ---- _needs_save = False + # Healthy-path saves only bump updated_at on one entry; they take + # the single-row UPSERT fast path instead of the full index rewrite + # (see _save_entry). Structural transitions (recover/create below) + # keep the full rewrite. + _metadata_only_save = False _needs_recover = False entry: Optional[SessionEntry] = None was_auto_reset = False @@ -2346,7 +2475,10 @@ def _get_or_create_session_impl( if session_key in self._entries and not force_new: entry = self._entries[session_key] - self._heal_compression_tip_locked( + # A heal rewrites entry.session_id, so it must reach the + # sessions.json mirror too: force the full-rewrite save + # below (the fast path persists state.db only). + _healed = self._heal_compression_tip_locked( entry, existing_session_id, canonical_existing_session_id ) @@ -2382,6 +2514,7 @@ def _get_or_create_session_impl( # window. Treat as healthy -- bump updated_at and save. entry.updated_at = now _needs_save = True + _metadata_only_save = not _healed else: # Stale check clean. Apply reset decision. if _reset_reason: @@ -2396,6 +2529,7 @@ def _get_or_create_session_impl( else: entry.updated_at = now _needs_save = True + _metadata_only_save = not _healed else: if not force_new: _needs_recover = True @@ -2462,7 +2596,10 @@ def _get_or_create_session_impl( } if _needs_save: - self._save_entries() + if _metadata_only_save: + self._save_entry(session_key) + else: + self._save_entries() # SQLite operations outside the lock (unchanged). if self._db and db_end_session_id: @@ -2506,19 +2643,28 @@ def update_session( """Update lightweight session metadata after an interaction.""" with self._lock: self._ensure_loaded_locked() - - if session_key in self._entries: - entry = self._entries[session_key] - entry.updated_at = _now() - if last_prompt_tokens is not None: - entry.last_prompt_tokens = last_prompt_tokens - self._save() - self._record_gateway_session_peer( - entry.session_id, - session_key, - entry.origin, - display_name=entry.display_name, - ) + entry = self._entries.get(session_key) + if entry is None: + return + entry.updated_at = _now() + if last_prompt_tokens is not None: + entry.last_prompt_tokens = last_prompt_tokens + # Snapshot peer fields while still holding _lock: a concurrent + # reset/heal may rewrite the entry, and mixing old and new + # fields would record a torn peer row. + peer_session_id = entry.session_id + peer_origin = entry.origin + peer_display_name = entry.display_name + # Metadata-only change on one entry: single-row UPSERT instead of + # the full index rewrite (see _save_entry). Both writes run outside + # ``_lock`` so the SQLite commit never blocks routing lookups. + self._save_entry(session_key) + self._record_gateway_session_peer( + peer_session_id, + session_key, + peer_origin, + display_name=peer_display_name, + ) def get_session_metadata( self, diff --git a/hermes_cli/auth.py b/hermes_cli/auth.py index c08c973045e8..82eb5ee7db5e 100644 --- a/hermes_cli/auth.py +++ b/hermes_cli/auth.py @@ -5741,6 +5741,18 @@ def _agent_key_is_usable(state: Dict[str, Any], min_ttl_seconds: int) -> bool: ) +# Per-process memo for resolve_nous_access_token. Startup runs +# check_tool_availability once per managed-tool check_fn (browser, image_gen, +# etc.), and each one independently triggers a ~15s blocking token-refresh +# network call when the stored token is expired. On a slow/constrained host that +# serial burst stretches startup to many minutes. A short-TTL memo collapses the +# burst into a single network round-trip; callers that need freshness use +# separate flows (force_fresh / refresh_nous_oauth_pure) and are unaffected. +_RESOLVE_TOKEN_CACHE_LOCK = threading.Lock() +_RESOLVE_TOKEN_CACHE: "tuple[float, str] | None" = None +_RESOLVE_TOKEN_CACHE_TTL_S = 5.0 + + def resolve_nous_access_token( *, timeout_seconds: float = 15.0, @@ -5749,6 +5761,16 @@ def resolve_nous_access_token( refresh_skew_seconds: int = ACCESS_TOKEN_REFRESH_SKEW_SECONDS, ) -> str: """Resolve a refresh-aware Nous Portal access token for managed tool gateways.""" + global _RESOLVE_TOKEN_CACHE + # Memo: collapse the startup burst of managed-tool check_fns into one + # network refresh. Only cache a successful, non-forced resolution for a + # short window; force_fresh / error paths bypass and don't populate it. + if not insecure and ca_bundle is None: + with _RESOLVE_TOKEN_CACHE_LOCK: + if _RESOLVE_TOKEN_CACHE is not None: + cached_at, cached_token = _RESOLVE_TOKEN_CACHE + if (time.monotonic() - cached_at) < _RESOLVE_TOKEN_CACHE_TTL_S: + return cached_token with _provider_state_transaction("nous") as ( auth_store, state, @@ -5803,6 +5825,15 @@ def resolve_nous_access_token( if not _is_expiring(state.get("expires_at"), refresh_skew_seconds): if merged_shared: _save_provider_state_to_source(auth_store, "nous", state, state_source_path) + # Populate the memo on the valid-token fast path too: the + # startup burst usually finds a *valid* token, but each + # check_fn call still pays two cross-process file locks and + # state reads to reach this return. The token has at least + # refresh_skew_seconds (>= 120s) of life here, so a 5s memo + # can never serve an expired token. + if not insecure and ca_bundle is None: + with _RESOLVE_TOKEN_CACHE_LOCK: + _RESOLVE_TOKEN_CACHE = (time.monotonic(), access_token) return access_token if not isinstance(refresh_token, str) or not refresh_token: @@ -5860,7 +5891,11 @@ def resolve_nous_access_token( } _save_provider_state_to_source(auth_store, "nous", state, state_source_path) _write_shared_nous_state(state) - return state["access_token"] + resolved = state["access_token"] + if not insecure and ca_bundle is None: + with _RESOLVE_TOKEN_CACHE_LOCK: + _RESOLVE_TOKEN_CACHE = (time.monotonic(), resolved) + return resolved def refresh_nous_oauth_pure( diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py index 4545575db26c..3181da0a3692 100644 --- a/hermes_cli/config_defaults.py +++ b/hermes_cli/config_defaults.py @@ -1587,6 +1587,18 @@ # a plugin in plugins/context_engine// or ~/.hermes/plugins/. "context": { "engine": "compressor", + # Return freed glibc allocator pages after long-running agent/TUI + # cleanup boundaries. Unsupported platforms are safe no-ops. + "memory_trim": { + "enabled": True, + "cooldown_seconds": 60.0, + # Successful trim calls are INFO logged every Nth periodic call; + # force paths always log so process-close behavior is visible. + "log_every_n": 1, + # Suppress INFO logs only when a readable RSS change is smaller. + # 0 reports every successful configured trim. + "info_log_min_delta_mb": 0.0, + }, }, # Persistent memory -- bounded curated memory injected into system prompt diff --git a/hermes_cli/gateway.py b/hermes_cli/gateway.py index 7d30531cf8df..681ce09c9ef1 100644 --- a/hermes_cli/gateway.py +++ b/hermes_cli/gateway.py @@ -1504,7 +1504,7 @@ def kill_gateway_processes( return killed -def _reap_unsupervised_gateway_orphans() -> bool: +def _reap_unsupervised_gateway_orphans(extra_exclude: set | None = None) -> bool: """Kill no-supervisor gateway orphans the pidfile/runtime record can't see. On WSL/no-systemd hosts the manual restart fallback runs the gateway @@ -1517,6 +1517,10 @@ def _reap_unsupervised_gateway_orphans() -> bool: running gateway — gating on ``supports_systemd_services()`` keeps the orphan-aware scan from killing live management processes there. + Args: + extra_exclude: Additional PIDs to skip (e.g. a PID already killed by + the caller so the sweep doesn't send a redundant SIGTERM/SIGKILL). + Returns True if at least one orphan was reaped. """ try: @@ -1528,6 +1532,8 @@ def _reap_unsupervised_gateway_orphans() -> bool: from gateway.status import _pid_exists, write_planned_stop_marker own = {os.getpid()} + if extra_exclude: + own |= extra_exclude try: # find_gateway_pids() includes no-supervisor `gateway restart` runtimes # for the current profile when no systemd supervisor is present. @@ -1583,6 +1589,12 @@ def stop_profile_gateway() -> bool: a live orphan still holds the webhook port. In that case fall back to the orphan-aware process scan so the replacement reaps the prior instance instead of stacking a duplicate on the same port (#51325). + + Even when the pid file is valid and points to the current gateway, older + orphans may linger from prior restarts that overwrote the pid file before + the old process exited. After killing the recorded PID, also sweep for + any remaining orphans so each restart produces at most one live gateway + (#75936). """ try: from gateway.status import get_running_pid, remove_pid_file @@ -1620,6 +1632,16 @@ def stop_profile_gateway() -> bool: if get_running_pid() is None: remove_pid_file() + + # Also reap any orphans from prior restarts whose PIDs were overwritten + # in the pid file before they exited (#75936). Exclude the PID we just + # killed so the sweep doesn't double-kill a process that's still tearing + # down — _reap_unsupervised_gateway_orphans already excludes our own PID. + try: + _reap_unsupervised_gateway_orphans(extra_exclude={pid} if pid else None) + except Exception as exc: + logger.debug("orphan reap after stop_profile_gateway failed: %s", exc) + return True diff --git a/hermes_cli/mem_trim.py b/hermes_cli/mem_trim.py new file mode 100644 index 000000000000..ad54f59a38e5 --- /dev/null +++ b/hermes_cli/mem_trim.py @@ -0,0 +1,255 @@ +"""Rate-limited heap release for long-lived Hermes gateway processes. + +On Linux/glibc, ``malloc_trim(0)`` can return pages from freed Python/C +allocations to the OS. Other platforms and allocators are safe no-ops. +Behavior is configured under ``context.memory_trim`` in ``config.yaml``. +""" + +from __future__ import annotations + +import ctypes +import gc +import logging +import platform +import sys +import threading +import time +from collections.abc import Callable +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + +_DEFAULT_COOLDOWN_SECONDS = 60.0 +_DEFAULT_LOG_EVERY_N = 1 +_DEFAULT_INFO_LOG_MIN_DELTA_MB = 0.0 +_trim_lock = threading.Lock() +_last_trim_monotonic = 0.0 +_probe_done = False +_malloc_trim: Callable[[int], int] | None = None +_trim_call_count = 0 + + +def _config_settings() -> tuple[bool, float, int, float]: + """Return fail-open settings from the normal Hermes config path.""" + enabled = True + cooldown: Any = _DEFAULT_COOLDOWN_SECONDS + log_every_n: Any = _DEFAULT_LOG_EVERY_N + info_log_min_delta_mb: Any = _DEFAULT_INFO_LOG_MIN_DELTA_MB + try: + # Read-only access: settings are only .get()ed and coerced, never + # mutated — use the no-deepcopy variant. This runs on EVERY trim + # attempt (before the cooldown check), and generating a full-config + # deepcopy per attempt is exactly the allocator garbage this module + # exists to release. + from hermes_cli.config import load_config_readonly + + config = load_config_readonly() or {} + context = config.get("context") if isinstance(config, dict) else None + settings = context.get("memory_trim") if isinstance(context, dict) else None + if isinstance(settings, dict): + configured_enabled = settings.get("enabled") + if isinstance(configured_enabled, bool): + enabled = configured_enabled + cooldown = settings.get("cooldown_seconds", _DEFAULT_COOLDOWN_SECONDS) + log_every_n = settings.get("log_every_n", _DEFAULT_LOG_EVERY_N) + info_log_min_delta_mb = settings.get( + "info_log_min_delta_mb", _DEFAULT_INFO_LOG_MIN_DELTA_MB + ) + except Exception: + pass + return ( + enabled, + _cooldown_seconds(cooldown), + _log_every_n(log_every_n), + _nonnegative_float(info_log_min_delta_mb, _DEFAULT_INFO_LOG_MIN_DELTA_MB), + ) + + +def _cooldown_seconds(value: Any) -> float: + if isinstance(value, bool): + return _DEFAULT_COOLDOWN_SECONDS + try: + return max(0.0, float(value)) + except (TypeError, ValueError): + return _DEFAULT_COOLDOWN_SECONDS + + +def _log_every_n(value: Any) -> int: + if isinstance(value, bool): + return _DEFAULT_LOG_EVERY_N + try: + return max(1, int(value)) + except (TypeError, ValueError): + return _DEFAULT_LOG_EVERY_N + + +def _nonnegative_float(value: Any, default: float) -> float: + if isinstance(value, bool): + return default + try: + return max(0.0, float(value)) + except (TypeError, ValueError): + return default + + +def _read_proc_status() -> str | None: + """Read Linux process status without making non-Linux callers special-case.""" + if sys.platform != "linux": + return None + try: + return Path("/proc/self/status").read_text(encoding="utf-8") + except OSError: + return None + + +def collect_memory_snapshot(history_bytes: int | None = None) -> dict[str, int | None]: + """Return lightweight process-memory telemetry for trim logs and canaries. + + ``VmRSS`` and ``RssAnon`` are Linux-only best effort fields. The helper is + intentionally dependency-free so allocation recovery never requires psutil. + """ + snapshot: dict[str, int | None] = { + "rss_kib": None, + "rss_anon_kib": None, + "thread_count": threading.active_count(), + } + status = _read_proc_status() + if status: + for line in status.splitlines(): + key, separator, raw_value = line.partition(":") + if not separator or key not in {"VmRSS", "RssAnon"}: + continue + value = raw_value.strip().split(maxsplit=1) + if value and value[0].isdigit(): + snapshot["rss_kib" if key == "VmRSS" else "rss_anon_kib"] = int(value[0]) + if isinstance(history_bytes, int) and history_bytes >= 0: + snapshot["history_bytes"] = history_bytes + return snapshot + + +def _should_log_trim( + *, force: bool, log_every_n: int, call_count: int, before: dict[str, int | None], + after: dict[str, int | None], info_log_min_delta_mb: float, +) -> bool: + # trim_memory calls this only after malloc_trim reported success. A forced + # successful trim is an explicit observability event, regardless of RSS. + if force: + return True + if not force and call_count % log_every_n: + return False + before_rss = before.get("rss_kib") + after_rss = after.get("rss_kib") + if before_rss is None or after_rss is None: + return True + return abs(after_rss - before_rss) >= info_log_min_delta_mb * 1024 + + +def _probe_glibc_malloc_trim() -> Callable[[int], int] | None: + """Resolve glibc's malloc_trim once; return None on unsupported systems.""" + global _malloc_trim, _probe_done + if _probe_done: + return _malloc_trim + _probe_done = True + if sys.platform != "linux": + return None + try: + if platform.libc_ver()[0].lower() != "glibc": + return None + libc = ctypes.CDLL(None) + trim = libc.malloc_trim + trim.argtypes = [ctypes.c_size_t] + trim.restype = ctypes.c_int + _malloc_trim = trim + except Exception as exc: + logger.debug("malloc_trim unavailable: %s", exc) + return _malloc_trim + + +def trim_memory( + *, + force: bool = False, + reason: str = "", + cooldown_seconds: float | None = None, +) -> bool: + """Collect cycles and ask glibc to release free heap pages. + + Returns ``True`` only when ``malloc_trim(0)`` ran and reported success. + Unsupported allocators, the config kill switch, cooldown suppression, and all + runtime errors return ``False`` without affecting the caller. + """ + ( + enabled, + configured_cooldown, + log_every_n, + info_log_min_delta_mb, + ) = _config_settings() + if not enabled: + return False + + global _last_trim_monotonic, _trim_call_count + with _trim_lock: + trim = _probe_glibc_malloc_trim() + if trim is None: + return False + now = time.monotonic() + cooldown = ( + configured_cooldown + if cooldown_seconds is None + else _cooldown_seconds(cooldown_seconds) + ) + if not force and _last_trim_monotonic and now - _last_trim_monotonic < cooldown: + return False + # Even forced trims honor a short floor: AIAgent.close() forces a trim, + # and delegate batches close N child subagents back-to-back in the SAME + # process — without a floor that stacks N+1 uncooled full gc.collect() + # passes (50-500ms each in a large gateway process). 5s coalesces the + # burst while keeping the parent's final close-trim effective. + _FORCE_FLOOR_SECONDS = 5.0 + if ( + force + and _last_trim_monotonic + and now - _last_trim_monotonic < _FORCE_FLOOR_SECONDS + ): + return False + # Record the attempt before calling into libc so repeated failures do not + # turn every turn boundary into an expensive full collection. + _last_trim_monotonic = now + try: + before = collect_memory_snapshot() + started = time.perf_counter() + gc.collect() + trim_result = trim(0) + released = bool(trim_result) + after = collect_memory_snapshot() + duration_ms = (time.perf_counter() - started) * 1000 + _trim_call_count += 1 + if released and _should_log_trim( + force=force, + log_every_n=log_every_n, + call_count=_trim_call_count, + before=before, + after=after, + info_log_min_delta_mb=info_log_min_delta_mb, + ): + logger.info( + "memory trim: reason=%s malloc_trim=%s rss_kib=%s->%s " + "rss_anon_kib=%s->%s threads=%s duration_ms=%.1f", + reason or "cleanup", + trim_result, + before.get("rss_kib"), + after.get("rss_kib"), + before.get("rss_anon_kib"), + after.get("rss_anon_kib"), + after.get("thread_count"), + duration_ms, + ) + return released + except Exception as exc: + logger.warning( + "memory trim failed after %s: %s: %s", + reason or "cleanup", + type(exc).__name__, + exc, + ) + return False diff --git a/hermes_cli/model_switch.py b/hermes_cli/model_switch.py index ef33260e2d5d..252986ed30ae 100644 --- a/hermes_cli/model_switch.py +++ b/hermes_cli/model_switch.py @@ -2702,6 +2702,7 @@ def _has_aws_sdk_creds_for_listing(slug: str) -> bool: live_models = fetch_api_models( api_key, api_url, + timeout=1.5 if for_picker else 5.0, # picker: fail fast so a slow custom endpoint doesn't block /model headers=_extra_headers_from_config(ep_cfg) or None, ) if live_models: @@ -2769,7 +2770,11 @@ def _has_aws_sdk_creds_for_listing(slug: str) -> bool: try: from hermes_cli.models import fetch_api_models - _live_models = fetch_api_models("", str(current_base_url).strip().rstrip("/")) + _live_models = fetch_api_models( + "", + str(current_base_url).strip().rstrip("/"), + timeout=1.5 if for_picker else 5.0, # picker: fail fast on a slow current endpoint + ) if _live_models: _models = _live_models except Exception: @@ -3012,6 +3017,7 @@ def _has_aws_sdk_creds_for_listing(slug: str) -> bool: live_models = fetch_api_models( api_key, api_url, + timeout=1.5 if for_picker else 5.0, # picker: fail fast so a slow custom endpoint doesn't block /model headers=grp.get("extra_headers") or None, ) if live_models: diff --git a/hermes_cli/tools_config.py b/hermes_cli/tools_config.py index 257eb0b72f6b..4d3a8c47530a 100644 --- a/hermes_cli/tools_config.py +++ b/hermes_cli/tools_config.py @@ -189,7 +189,11 @@ def _xai_credentials_present() -> bool: return True except Exception: pass - return bool(str(os.environ.get("XAI_API_KEY") or "").strip()) + try: + from agent.secret_scope import get_secret + except ImportError: # pragma: no cover — secret_scope is in-repo + return bool(str(os.environ.get("XAI_API_KEY") or "").strip()) + return bool(str(get_secret("XAI_API_KEY") or "").strip()) def _homeassistant_credentials_present() -> bool: diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index 2f4ace574b10..5af536fdce13 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -2911,6 +2911,62 @@ def _collect_profile_gateway_topology() -> Dict[str, Any]: return {"profiles": profile_names, "gateway_mode": mode, "gateways": gateways} +# /api/status is polled ~1/s by the desktop app while it waits for the backend +# (and again by the dashboard badge). Each uncached call above walks 7+ profile +# homes (yaml.safe_load with the pure-Python loader + psutil process-table +# probes + realpath walks) inside the default executor; concurrent polls pile +# up and hold the GIL for 14-16s, starving the event loop — the desktop WS +# never receives gateway.ready and boot fails ("event loop stalled ... GIL +# pressure suspected"). Topology changes on gateway start/stop, so a short TTL +# cache with a collapse lock keeps the scan to one per window. The cache also +# remembers which collector produced the entry: tests monkeypatch +# _collect_profile_gateway_topology per case, and the identity check keeps +# them hermetic without needing a reset hook (a swapped collector is a miss). +_TOPOLOGY_CACHE: Dict[str, Any] = {"ts": 0.0, "data": None, "fn": None} +_TOPOLOGY_CACHE_LOCK = threading.Lock() +_TOPOLOGY_CACHE_TTL = 10.0 + + +def _topology_cache_get(fn: Any) -> Optional[Dict[str, Any]]: + if ( + _TOPOLOGY_CACHE["data"] is not None + and _TOPOLOGY_CACHE["fn"] is fn + and time.monotonic() - _TOPOLOGY_CACHE["ts"] < _TOPOLOGY_CACHE_TTL + ): + return _TOPOLOGY_CACHE["data"] + return None + + +def _collect_profile_gateway_topology_cached() -> Dict[str, Any]: + fn = _collect_profile_gateway_topology + cached = _topology_cache_get(fn) + if cached is not None: + return cached + with _TOPOLOGY_CACHE_LOCK: + cached = _topology_cache_get(fn) + if cached is not None: + return cached + data = fn() + _TOPOLOGY_CACHE["data"] = data + _TOPOLOGY_CACHE["fn"] = fn + _TOPOLOGY_CACHE["ts"] = time.monotonic() + return data + + +def _load_configured_gateway_platforms() -> set[str]: + """Load connected platform names away from the asyncio event loop. + + The first ``load_gateway_config()`` call performs platform discovery and + can take longer than Desktop's WebSocket connect timeout on Windows. This + helper is synchronous by design; ``get_status`` runs it in Starlette's + worker pool so a concurrent ``/api/ws`` handshake can still complete. + """ + from gateway.config import load_gateway_config + + gateway_config = load_gateway_config() + return {platform.value for platform in gateway_config.get_connected_platforms()} + + @app.get("/api/ssh/ownership") async def get_ssh_ownership(request: Request): _require_token(request) @@ -3012,12 +3068,9 @@ def _bounded_health_probe(): gateway_updated_at = None configured_gateway_platforms: set[str] | None = None try: - from gateway.config import load_gateway_config - - gateway_config = load_gateway_config() - configured_gateway_platforms = { - platform.value for platform in gateway_config.get_connected_platforms() - } + configured_gateway_platforms = await run_in_threadpool( + _load_configured_gateway_platforms + ) except Exception: configured_gateway_platforms = None @@ -3237,7 +3290,7 @@ def _bounded_health_probe(): # per-gateway ``gateways[]`` detail carries host ports (deployment # recon), so it stays gated with the host paths / PID below. topology = await asyncio.get_running_loop().run_in_executor( - None, _collect_profile_gateway_topology + None, _collect_profile_gateway_topology_cached ) status["profiles"] = topology["profiles"] status["gateway_mode"] = topology["gateway_mode"] @@ -4320,7 +4373,20 @@ async def get_elevenlabs_voices(profile: Optional[str] = None): # Config-only scope (await-safe): the key lookup reads the requested # profile's .env, matching the profile the settings UI writes to. with _config_profile_scope(profile): - api_key = (load_env().get("ELEVENLABS_API_KEY") or os.environ.get("ELEVENLABS_API_KEY") or "").strip() + api_key = (load_env().get("ELEVENLABS_API_KEY") or "").strip() + if not api_key: + # Fallback for env-only deployments — scope-aware (Slack pattern): + # under multiplex os.environ may hold another profile's key, so + # honor the installed scope's verdict before touching the env. + try: + from agent.secret_scope import UnscopedSecretError, get_secret + + try: + api_key = (get_secret("ELEVENLABS_API_KEY") or "").strip() + except UnscopedSecretError: + api_key = (os.environ.get("ELEVENLABS_API_KEY") or "").strip() + except Exception: + api_key = (os.environ.get("ELEVENLABS_API_KEY") or "").strip() if not api_key: return {"available": False, "voices": []} diff --git a/hermes_state.py b/hermes_state.py index 71a77587fb62..39ec0806fe84 100644 --- a/hermes_state.py +++ b/hermes_state.py @@ -3599,6 +3599,92 @@ def get_compression_failure_cooldown( "error": error, } + def get_compression_failure_cooldown_row( + self, + session_id: str, + ) -> Dict[str, Any]: + """Return the exact stored cooldown columns without expiry filtering. + + Compression cancellation uses this under its session lease so rollback + can preserve an expired row, a partially-null row, or an absent session + exactly instead of converting those states through the active-cooldown + API. + """ + if not session_id: + return {"session_exists": False, "cooldown_until": None, "error": None} + with self._lock: + row = self._conn.execute( + "SELECT compression_failure_cooldown_until, compression_failure_error " + "FROM sessions WHERE id = ?", + (session_id,), + ).fetchone() + if row is None: + return {"session_exists": False, "cooldown_until": None, "error": None} + cooldown_until = ( + row["compression_failure_cooldown_until"] + if isinstance(row, sqlite3.Row) + else row[0] + ) + error = ( + row["compression_failure_error"] + if isinstance(row, sqlite3.Row) + else row[1] + ) + return { + "session_exists": True, + "cooldown_until": ( + float(cooldown_until) if cooldown_until is not None else None + ), + "error": error, + } + + def restore_compression_failure_cooldown_row( + self, + session_id: str, + snapshot: Dict[str, Any], + ) -> None: + """Restore and verify an exact cooldown-row snapshot. + + Unlike the ordinary record/clear helpers, this transactional rollback + API deliberately propagates write and verification failures. A caller + must not report cancellation as mutation-free when compensation failed. + """ + expected_exists = bool(snapshot.get("session_exists", False)) + if not expected_exists: + actual = self.get_compression_failure_cooldown_row(session_id) + if actual.get("session_exists", False): + raise RuntimeError( + "cannot restore absent compression cooldown row: session now exists" + ) + return + + deadline = snapshot.get("cooldown_until") + error = snapshot.get("error") + + def _do(conn): + cursor = conn.execute( + "UPDATE sessions SET compression_failure_cooldown_until = ?, " + "compression_failure_error = ? WHERE id = ?", + (deadline, error, session_id), + ) + if cursor.rowcount != 1: + raise RuntimeError( + f"compression cooldown rollback session missing: {session_id}" + ) + + self._execute_write(_do) + actual = self.get_compression_failure_cooldown_row(session_id) + expected = { + "session_exists": True, + "cooldown_until": float(deadline) if deadline is not None else None, + "error": error, + } + if actual != expected: + raise RuntimeError( + f"compression cooldown rollback verification failed: " + f"expected={expected!r}, actual={actual!r}" + ) + def clear_compression_failure_cooldown(self, session_id: str) -> None: """Clear any persisted compression-failure cooldown for a session.""" if not session_id: diff --git a/mini_swe_runner.py b/mini_swe_runner.py index 2853abc9a01e..2994bfc6d25a 100644 --- a/mini_swe_runner.py +++ b/mini_swe_runner.py @@ -467,7 +467,7 @@ def run_task(self, task: str) -> Dict[str, Any]: response = self.client.chat.completions.create(**api_kwargs) except Exception as e: - self.logger.error(f"API call failed: {e}") + self.logger.error("API call failed: %s", e) break assistant_message = response.choices[0].message @@ -607,7 +607,7 @@ def run_batch( print(f"✅ Task {i} completed (api_calls={result['api_calls']})") except Exception as e: - self.logger.error(f"Error on task {i}: {e}") + self.logger.error("Error on task %s: %s", i, e) error_result = { "conversations": [], "completed": False, diff --git a/plugins/browser/browser_use/provider.py b/plugins/browser/browser_use/provider.py index e098cf814a65..a10d9a361363 100644 --- a/plugins/browser/browser_use/provider.py +++ b/plugins/browser/browser_use/provider.py @@ -37,6 +37,7 @@ import requests from agent.browser_provider import BrowserProvider +from agent.secret_scope import get_secret logger = logging.getLogger(__name__) @@ -137,7 +138,7 @@ def _get_config_or_none(self, *, refresh_token: bool = True) -> Optional[Dict[st # Direct API key wins unless the user has explicitly opted into the # managed Nous gateway via ``tool_gateway.browser: gateway``. - api_key = os.environ.get("BROWSER_USE_API_KEY") + api_key = get_secret("BROWSER_USE_API_KEY") if api_key and not prefers_gateway("browser"): return { "api_key": api_key, diff --git a/plugins/browser/browserbase/provider.py b/plugins/browser/browserbase/provider.py index c828ae29b2da..04829f740f53 100644 --- a/plugins/browser/browserbase/provider.py +++ b/plugins/browser/browserbase/provider.py @@ -39,6 +39,7 @@ import requests from agent.browser_provider import BrowserProvider +from agent.secret_scope import get_secret logger = logging.getLogger(__name__) @@ -66,8 +67,8 @@ def is_available(self) -> bool: # ------------------------------------------------------------------ def _get_config_or_none(self) -> Optional[Dict[str, Any]]: - api_key = os.environ.get("BROWSERBASE_API_KEY") - project_id = os.environ.get("BROWSERBASE_PROJECT_ID") + api_key = get_secret("BROWSERBASE_API_KEY") + project_id = get_secret("BROWSERBASE_PROJECT_ID") if api_key and project_id: return { "api_key": api_key, diff --git a/plugins/browser/firecrawl/provider.py b/plugins/browser/firecrawl/provider.py index 892cd6c1dc8f..2177cd96e2ff 100644 --- a/plugins/browser/firecrawl/provider.py +++ b/plugins/browser/firecrawl/provider.py @@ -34,6 +34,7 @@ import requests from agent.browser_provider import BrowserProvider +from agent.secret_scope import get_secret logger = logging.getLogger(__name__) @@ -56,7 +57,7 @@ def display_name(self) -> str: return "Firecrawl" def is_available(self) -> bool: - return bool(os.environ.get("FIRECRAWL_API_KEY")) + return bool(get_secret("FIRECRAWL_API_KEY")) # ------------------------------------------------------------------ # Session lifecycle @@ -66,7 +67,7 @@ def _api_url(self) -> str: return os.environ.get("FIRECRAWL_API_URL", _BASE_URL) def _headers(self) -> Dict[str, str]: - api_key = os.environ.get("FIRECRAWL_API_KEY") + api_key = get_secret("FIRECRAWL_API_KEY") if not api_key: raise ValueError( "FIRECRAWL_API_KEY environment variable is required. " diff --git a/plugins/google_meet/meet_bot.py b/plugins/google_meet/meet_bot.py index 0396d039490a..20745eb3de37 100644 --- a/plugins/google_meet/meet_bot.py +++ b/plugins/google_meet/meet_bot.py @@ -456,6 +456,10 @@ def run_bot() -> int: # noqa: C901 — orchestration, explicit branches realtime_model = os.environ.get("HERMES_MEET_REALTIME_MODEL", "gpt-realtime") realtime_voice = os.environ.get("HERMES_MEET_REALTIME_VOICE", "alloy") realtime_instructions = os.environ.get("HERMES_MEET_REALTIME_INSTRUCTIONS", "") + # HERMES_MEET_REALTIME_KEY is set explicitly by process_manager.start(), + # which resolves it through the parent's profile secret scope at spawn + # time. The bare OPENAI_API_KEY fallback only serves standalone + # `python -m plugins.google_meet.meet_bot` runs outside the gateway. realtime_api_key = os.environ.get("HERMES_MEET_REALTIME_KEY") or os.environ.get("OPENAI_API_KEY", "") if not url or not _is_safe_meet_url(url): diff --git a/plugins/google_meet/process_manager.py b/plugins/google_meet/process_manager.py index 0709c6a1f944..4b2576efb486 100644 --- a/plugins/google_meet/process_manager.py +++ b/plugins/google_meet/process_manager.py @@ -152,6 +152,22 @@ def start( env["HERMES_MEET_REALTIME_VOICE"] = realtime_voice if realtime_instructions: env["HERMES_MEET_REALTIME_INSTRUCTIONS"] = realtime_instructions + # Resolve the realtime key at SPAWN time, in the parent, where the + # profile secret scope (a contextvar) is still installed. The detached + # child inherits the process environment — NOT the scope — so under a + # multiplexed gateway an in-child os.environ read would see another + # profile's OPENAI_API_KEY (or nothing). Pass it explicitly instead; + # meet_bot checks HERMES_MEET_REALTIME_KEY before OPENAI_API_KEY. + if not realtime_api_key: + try: + from agent.secret_scope import get_secret + + realtime_api_key = ( + get_secret("HERMES_MEET_REALTIME_KEY") + or get_secret("OPENAI_API_KEY") + ) + except ImportError: # pragma: no cover — secret_scope is in-repo + pass if realtime_api_key: env["HERMES_MEET_REALTIME_KEY"] = realtime_api_key diff --git a/plugins/image_gen/deepinfra/__init__.py b/plugins/image_gen/deepinfra/__init__.py index 1ef5dcfb6c7e..90754b369f76 100644 --- a/plugins/image_gen/deepinfra/__init__.py +++ b/plugins/image_gen/deepinfra/__init__.py @@ -31,6 +31,7 @@ import os from typing import Any, Dict, List, Optional +from agent.secret_scope import get_secret from agent.image_gen_provider import ( DEFAULT_ASPECT_RATIO, ImageGenProvider, @@ -138,7 +139,7 @@ def display_name(self) -> str: return "DeepInfra" def is_available(self) -> bool: - return bool(os.environ.get("DEEPINFRA_API_KEY", "").strip()) + return bool((get_secret("DEEPINFRA_API_KEY", "") or "").strip()) def list_models(self) -> List[Dict[str, Any]]: live = _live_models() @@ -199,7 +200,7 @@ def generate( aspect_ratio=aspect, ) - api_key = os.environ.get("DEEPINFRA_API_KEY", "").strip() + api_key = (get_secret("DEEPINFRA_API_KEY", "") or "").strip() if not api_key: return error_response( error=( diff --git a/plugins/image_gen/krea/__init__.py b/plugins/image_gen/krea/__init__.py index d7b260667aa6..64c53c447f17 100644 --- a/plugins/image_gen/krea/__init__.py +++ b/plugins/image_gen/krea/__init__.py @@ -30,6 +30,7 @@ import requests +from agent.secret_scope import get_secret from agent.image_gen_provider import ( DEFAULT_ASPECT_RATIO, ImageGenProvider, @@ -177,7 +178,7 @@ def _resolve_managed_krea_gateway(): logger.debug("Managed Krea gateway resolution unavailable: %s", exc) return None - if os.environ.get("KREA_API_KEY") and not prefers_gateway("image_gen"): + if get_secret("KREA_API_KEY") and not prefers_gateway("image_gen"): return None try: @@ -233,7 +234,7 @@ def is_available(self) -> bool: # Available with a direct Krea key OR via the managed Nous gateway # (Nous Subscription), so portal users with no Krea key can still # reach Krea 2 through the gateway. - return bool(os.environ.get("KREA_API_KEY")) or _managed_krea_gateway_ready() + return bool(get_secret("KREA_API_KEY")) or _managed_krea_gateway_ready() def list_models(self) -> List[Dict[str, Any]]: return [ @@ -338,7 +339,7 @@ def generate( auth_token = managed.nous_user_token else: base_url = BASE_URL - auth_token = os.environ.get("KREA_API_KEY") + auth_token = get_secret("KREA_API_KEY") if not auth_token: return error_response( error=( diff --git a/plugins/image_gen/openai/__init__.py b/plugins/image_gen/openai/__init__.py index cfa9e42c908f..6fc4bc58e04b 100644 --- a/plugins/image_gen/openai/__init__.py +++ b/plugins/image_gen/openai/__init__.py @@ -27,6 +27,7 @@ import os from typing import Any, Dict, List, Optional, Tuple +from agent.secret_scope import get_secret from agent.image_gen_provider import ( DEFAULT_ASPECT_RATIO, ImageGenProvider, @@ -173,7 +174,7 @@ def display_name(self) -> str: return "OpenAI" def is_available(self) -> bool: - if not os.environ.get("OPENAI_API_KEY"): + if not get_secret("OPENAI_API_KEY"): return False try: import openai # noqa: F401 @@ -235,7 +236,8 @@ def generate( aspect_ratio=aspect, ) - if not os.environ.get("OPENAI_API_KEY"): + api_key = get_secret("OPENAI_API_KEY") + if not api_key: return error_response( error=( "OPENAI_API_KEY not set. Run `hermes tools` → Image " @@ -270,7 +272,7 @@ def generate( is_edit = bool(sources) modality = "image" if is_edit else "text" - client = openai.OpenAI() + client = openai.OpenAI(api_key=api_key) if is_edit: # images.edit() expects file-like objects. Download/read each diff --git a/plugins/memory/hindsight/__init__.py b/plugins/memory/hindsight/__init__.py index 03666f2c4180..086bb1239bcb 100644 --- a/plugins/memory/hindsight/__init__.py +++ b/plugins/memory/hindsight/__init__.py @@ -39,10 +39,13 @@ import queue import sys import threading +import time from datetime import datetime, timezone from typing import Any, Dict, List +from agent.secret_scope import get_secret + from agent.memory_provider import MemoryProvider from hermes_constants import get_hermes_home from tools.registry import tool_error @@ -383,7 +386,7 @@ def _load_config() -> dict: return { "mode": os.environ.get("HINDSIGHT_MODE", "cloud"), - "apiKey": os.environ.get("HINDSIGHT_API_KEY", ""), + "apiKey": get_secret("HINDSIGHT_API_KEY", ""), "timeout": _parse_int_setting(os.environ.get("HINDSIGHT_TIMEOUT"), _DEFAULT_TIMEOUT), "idle_timeout": _parse_int_setting(os.environ.get("HINDSIGHT_IDLE_TIMEOUT"), _DEFAULT_IDLE_TIMEOUT), "retain_tags": os.environ.get("HINDSIGHT_RETAIN_TAGS", ""), @@ -522,7 +525,7 @@ def _build_embedded_profile_env(config: dict[str, Any], *, llm_api_key: str | No current_key = ( config.get("llmApiKey") or config.get("llm_api_key") - or os.environ.get("HINDSIGHT_LLM_API_KEY", "") + or get_secret("HINDSIGHT_LLM_API_KEY", "") ) current_provider = config.get("llm_provider", "") @@ -726,6 +729,21 @@ def __init__(self): self._writer_thread: threading.Thread | None = None self._shutting_down = threading.Event() self._atexit_registered = False + # Server-side async retain operations still in flight. With + # retain_async=True, aretain_batch returns as soon as the write is + # *accepted*, not when it's durable/recall-visible, so the returned + # operation_id(s) stay "pending" until the server finishes. The + # background prefetch gates on these via get_operation_status so recall + # observes the just-completed turn (draining the local queue alone is + # not a read-after-write signal for async retains). + self._pending_retain_ops: set[str] = set() + self._pending_retain_ops_lock = threading.Lock() + self._retain_ops_bank_id = "" + # Seconds between get_operation_status polls while waiting for server- + # side retain completion. Each poll is a server round trip, so this is + # deliberately coarser than the 0.05s local queue-drain poll: ~20 calls + # max over the default 10s budget instead of ~200. + self._RETAIN_OP_POLL_INTERVAL_S = 0.5 # Legacy alias — older tests/callers reference _sync_thread directly. # Points at _writer_thread once the writer is running. self._sync_thread = None @@ -742,6 +760,17 @@ def __init__(self): self._auto_retain = True self._retain_every_n_turns = 1 self._retain_async = True + # Async retain never blocks the reply (writes drain on the single + # writer thread). But the next turn's warm prefetch runs on its own + # thread and could read BEFORE the just-completed retain is + # recall-visible on the server, dropping the latest turn from recall. + # When True, the background prefetch first waits (bounded) for the + # local writer queue to drain AND for the server-side async retain + # operation(s) to report completion, an explicit read-after-write + # signal — closing that race without putting any write on the reply + # path. + self._prefetch_waits_for_retain = True + self._prefetch_retain_drain_timeout = 10.0 self._retain_context = "conversation between Hermes Agent and the User" self._turn_counter = 0 self._session_turns: list[str] = [] # accumulates ALL turns for the session @@ -786,7 +815,7 @@ def is_available(self) -> bool: has_key = bool( cfg.get("apiKey") or cfg.get("api_key") - or os.environ.get("HINDSIGHT_API_KEY", "") + or get_secret("HINDSIGHT_API_KEY", "") ) has_url = bool(cfg.get("api_url") or os.environ.get("HINDSIGHT_API_URL", "")) return has_key or has_url @@ -895,7 +924,7 @@ def post_setup(self, hermes_home: str, config: dict) -> None: # Step 3: Mode-specific config if mode == "cloud": print("\n Get your API key at https://ui.hindsight.vectorize.io\n") - existing_key = os.environ.get("HINDSIGHT_API_KEY", "") + existing_key = get_secret("HINDSIGHT_API_KEY", "") or "" if existing_key: masked = f"...{existing_key[-4:]}" if len(existing_key) > 4 else "set" sys.stdout.write(f" API key (current: {masked}, blank to keep): ") @@ -1058,6 +1087,8 @@ def get_config_schema(self): {"key": "auto_retain", "description": "Automatically retain conversation turns", "default": True}, {"key": "retain_every_n_turns", "description": "Retain every N turns (1 = every turn)", "default": 1}, {"key": "retain_async","description": "Process retain asynchronously on the Hindsight server", "default": True}, + {"key": "prefetch_waits_for_retain", "description": "Have the background next-turn prefetch wait for the just-completed retain to become recall-visible on the server (local queue drain + async operation completion) before recalling, so recall includes the just-completed turn (runs off the reply path, adds no response latency)", "default": True}, + {"key": "prefetch_retain_drain_timeout", "description": "Max seconds the background prefetch waits for the retain to become recall-visible (queue drain + server-side completion) before recalling anyway", "default": 10.0}, {"key": "retain_context", "description": "Context label for retained memories", "default": "conversation between Hermes Agent and the User"}, {"key": "recall_max_tokens", "description": "Maximum tokens for recall results", "default": 4096}, {"key": "recall_max_input_chars", "description": "Maximum input query length for auto-recall", "default": 800}, @@ -1094,7 +1125,7 @@ def _get_client(self): kwargs = dict( profile=self._config.get("profile", "hermes"), llm_provider=llm_provider, - llm_api_key=self._config.get("llmApiKey") or self._config.get("llm_api_key") or os.environ.get("HINDSIGHT_LLM_API_KEY", ""), + llm_api_key=self._config.get("llmApiKey") or self._config.get("llm_api_key") or get_secret("HINDSIGHT_LLM_API_KEY", ""), llm_model=self._config.get("llm_model", ""), ) if self._llm_base_url: @@ -1162,6 +1193,169 @@ def _ensure_writer(self) -> None: self._sync_thread = thread thread.start() + def _track_retain_ops(self, retain_response, bank_id: str) -> None: + """Record server-side async operation id(s) from an aretain_batch reply. + + Async retains return ``operation_id`` / ``operation_ids`` that stay + ``pending`` on the server until the write is durable and recall-visible. + The bank_id is captured alongside so completion can be polled with the + same bank the write targeted. + """ + ids: list[str] = [] + single = getattr(retain_response, "operation_id", None) + if single: + ids.append(str(single)) + multiple = getattr(retain_response, "operation_ids", None) + if multiple: + ids.extend(str(op) for op in multiple if op) + if not ids: + # Server didn't hand back an op id (older API, or it completed + # synchronously). Nothing to poll — local queue drain is the only + # available signal in that case. + return + self._retain_ops_bank_id = bank_id + with self._pending_retain_ops_lock: + self._pending_retain_ops.update(ids) + + def _is_retain_op_complete(self, bank_id: str, op_id: str) -> bool: + """Return True when a server-side async retain op is done (or gone). + + ``get_operation_status`` returns ``completed``/``failed`` for a known + op; completed ops are evicted server-side, so a NotFound (404) also + means "no longer pending" and is treated as done. Transient errors + return False so the caller keeps waiting until its deadline. + """ + from hindsight_client_api.exceptions import NotFoundException + + try: + resp = self._run_hindsight_operation( + lambda client: client.operations.get_operation_status( + bank_id=bank_id, operation_id=op_id + ) + ) + except NotFoundException: + return True + except Exception as exc: + logger.debug("Prefetch: operation status check failed for %s: %s", op_id, exc) + return False + status = str(getattr(resp, "status", "") or "").lower() + return status in {"completed", "failed"} + + def _wait_for_retains_drained(self, timeout: float) -> bool: + """Block up to *timeout* seconds for the just-completed turn's retain to + become recall-visible on the server. + + Used by the background prefetch so the next turn's recall observes the + just-completed turn's write instead of racing ahead of it. Runs only on + the background prefetch thread — never on the reply path. + + Two ordered barriers, both bounded by the shared *timeout* budget: + + 1. Local writer queue drains (the retain call has been *dispatched* to + the server). Polls ``unfinished_tasks`` rather than ``queue.join()`` + so a wedged write can't hang the prefetch. + 2. Server-side async operations complete. With ``retain_async=True`` the + dispatched call returns on *acceptance*, not durability, so draining + the local queue alone is NOT a read-after-write signal. We poll + ``get_operation_status`` for the tracked op id(s) until the server + reports completion (an explicit read-after-write condition). + + Returns True if both barriers cleared within the budget, False on + timeout/shutdown. + """ + deadline = None if timeout <= 0 else time.monotonic() + timeout + + def _expired() -> bool: + return deadline is not None and time.monotonic() >= deadline + + # Barrier 1: local queue drain (retain dispatched to the server). + while self._retain_queue.unfinished_tasks > 0: + if self._shutting_down.is_set(): + return False + if _expired(): + logger.debug( + "Prefetch: retain drain timed out after %.1fs (%d pending)", + timeout, self._retain_queue.unfinished_tasks, + ) + return False + time.sleep(0.05) + + # Barrier 2: server-side async retain completion (read-after-write). + return self._wait_for_server_retain_ops(deadline, timeout) + + def _wait_for_server_retain_ops(self, deadline: float | None, timeout: float) -> bool: + """Poll tracked async retain ops until complete or the deadline passes. + + *deadline* is a ``time.monotonic()`` value (None = no bound). Completed + ops are removed from the pending set as they finish so a later prefetch + doesn't re-poll them. + + Ops still pending when the deadline expires are DROPPED, not retained: + keeping them would make a permanently failing status endpoint (auth + error, endless 500s, server that loses ops without a 404) grow the + pending set forever and burn the full timeout on EVERY subsequent + prefetch — turning "bounded wait per prefetch" into unbounded + session-wide degradation (and, via prefetch()'s bounded join on the + reply path, a per-turn reply-latency penalty). Dropping trades a + possibly-stale recall NOW (identical to prefetch_waits_for_retain=False + behavior) for guaranteed liveness; the drop is logged at WARNING once + per prefetch so persistent server trouble is visible. + + Status polls are spaced by _RETAIN_OP_POLL_INTERVAL_S (0.5s) — server + round trips per op are bounded (~20 over a 10s budget), unlike the + cheap 0.05s local queue-drain poll in _wait_for_retains_drained. + """ + while True: + with self._pending_retain_ops_lock: + bank_id = getattr(self, "_retain_ops_bank_id", "") or self._bank_id + pending = list(self._pending_retain_ops) + if not pending: + return True + if self._shutting_down.is_set(): + return False + + done: set[str] = set() + expired = False + for op_id in pending: + if self._shutting_down.is_set(): + return False + if deadline is not None and time.monotonic() >= deadline: + expired = True + break + if self._is_retain_op_complete(bank_id, op_id): + done.add(op_id) + + if expired: + with self._pending_retain_ops_lock: + self._pending_retain_ops.difference_update(done) + dropped = len(self._pending_retain_ops) + self._pending_retain_ops.clear() + logger.warning( + "Prefetch: server retain visibility timed out after %.1fs; " + "dropping %d unresolved op(s) so later prefetches stay " + "bounded (recall may miss the just-completed turn)", + timeout, dropped, + ) + return False + + with self._pending_retain_ops_lock: + self._pending_retain_ops.difference_update(done) + still_pending = bool(self._pending_retain_ops) + if not still_pending: + return True + if deadline is not None and time.monotonic() >= deadline: + with self._pending_retain_ops_lock: + dropped = len(self._pending_retain_ops) + self._pending_retain_ops.clear() + logger.warning( + "Prefetch: server retain visibility timed out after %.1fs; " + "dropping %d unresolved op(s) so later prefetches stay " + "bounded (recall may miss the just-completed turn)", + timeout, dropped, + ) + return False + time.sleep(self._RETAIN_OP_POLL_INTERVAL_S) + def _writer_loop(self) -> None: """Drain the retain queue serially. Exits on sentinel. @@ -1330,7 +1524,7 @@ def initialize(self, session_id: str, **kwargs) -> None: ) self._mode = "disabled" return - self._api_key = self._config.get("apiKey") or self._config.get("api_key") or os.environ.get("HINDSIGHT_API_KEY", "") + self._api_key = self._config.get("apiKey") or self._config.get("api_key") or get_secret("HINDSIGHT_API_KEY", "") default_url = _DEFAULT_LOCAL_URL if self._mode in {"local_embedded", "local_external"} else _DEFAULT_API_URL self._api_url = self._config.get("api_url") or os.environ.get("HINDSIGHT_API_URL", default_url) self._llm_base_url = self._config.get("llm_base_url", "") @@ -1404,6 +1598,10 @@ def initialize(self, session_id: str, **kwargs) -> None: self._recall_prompt_preamble = self._config.get("recall_prompt_preamble", "") self._recall_max_input_chars = int(self._config.get("recall_max_input_chars", 800)) self._retain_async = self._config.get("retain_async", True) + self._prefetch_waits_for_retain = self._config.get("prefetch_waits_for_retain", True) + self._prefetch_retain_drain_timeout = float( + self._config.get("prefetch_retain_drain_timeout", 10.0) + ) _client_version = "unknown" try: @@ -1548,6 +1746,15 @@ def queue_prefetch(self, query: str, *, session_id: str = "") -> None: query = query[:self._recall_max_input_chars] def _run(): + # Ensure the just-completed turn's retain is recall-visible on the + # server before we recall, so the warmed context for the next turn + # includes it. This waits for the local writer queue to drain AND + # for the server-side async retain op(s) to complete (an explicit + # read-after-write signal), because async retain returns on + # acceptance rather than durability. Runs on the background prefetch + # thread, never the reply path, so it adds no response latency. + if self._prefetch_waits_for_retain: + self._wait_for_retains_drained(self._prefetch_retain_drain_timeout) try: if self._prefetch_method == "reflect": logger.debug("Prefetch: calling reflect (bank=%s, query_len=%d)", self._bank_id, len(query)) @@ -1729,7 +1936,7 @@ def _do_retain() -> None: item["update_mode"] = update_mode logger.debug("Hindsight retain: bank=%s, doc=%s, mode=%s, async=%s, content_len=%d, num_turns=%d", bank_id, document_id, update_mode, retain_async_flag, len(content), num_turns) - self._run_hindsight_operation( + resp = self._run_hindsight_operation( lambda client: client.aretain_batch( bank_id=bank_id, items=[item], @@ -1737,6 +1944,11 @@ def _do_retain() -> None: retain_async=retain_async_flag, ) ) + # For async retains the write is only *accepted* here; track the + # returned operation id(s) so the next-turn prefetch can wait for + # true server-side completion (read-after-write) before recalling. + if retain_async_flag: + self._track_retain_ops(resp, bank_id) logger.debug("Hindsight retain succeeded") self._ensure_writer() diff --git a/plugins/memory/holographic/holographic.py b/plugins/memory/holographic/holographic.py index e1401fde108d..31c94e10f51d 100644 --- a/plugins/memory/holographic/holographic.py +++ b/plugins/memory/holographic/holographic.py @@ -33,6 +33,7 @@ logger = logging.getLogger(__name__) _TWO_PI = 2.0 * math.pi +_FLOAT32_BLOB_PREFIX = b"HRR1" def _require_numpy() -> None: @@ -40,6 +41,12 @@ def _require_numpy() -> None: raise RuntimeError("numpy is required for holographic operations") +def _np(): + """Return the numpy module after the runtime availability guard.""" + _require_numpy() + return np # type: ignore[name-defined] + + def encode_atom(word: str, dim: int = 1024) -> "np.ndarray": """Deterministic phase vector via SHA-256 counter blocks. @@ -160,20 +167,100 @@ def encode_fact(content: str, entities: list[str], dim: int = 1024) -> "np.ndarr return bundle(*components) -def phases_to_bytes(phases: "np.ndarray") -> bytes: - """Serialize phase vector to bytes. float64 tobytes — 8 KB at dim=1024.""" - _require_numpy() - return phases.tobytes() - +def phases_to_bytes(phases: "np.ndarray", dim: int | None = None) -> bytes: + """Serialize phase vectors as float32 blobs. -def bytes_to_phases(data: bytes) -> "np.ndarray": - """Deserialize bytes back to phase vector. Inverse of phases_to_bytes. + float32 halves SQLite BLOB storage versus the legacy float64 format + (4 KB + a 4-byte format prefix instead of 8 KB at dim=1024) while + preserving enough precision for phase-similarity retrieval. + ``bytes_to_phases`` keeps reading legacy float64 blobs for backward + compatibility. - The .copy() call is required because frombuffer returns a read-only view - backed by the bytes object; callers expect a mutable array. + When ``dim`` is 1 the prefixed float32 blob (8 bytes) collides in size + with a raw float64 blob (8 bytes), making the format ambiguous. In + that case we fall back to writing raw float64 so that ``bytes_to_phases`` + can never misinterpret the blob. """ - _require_numpy() - return np.frombuffer(data, dtype=np.float64).copy() + numpy = _np() + if dim is None: + dim = int(phases.shape[0]) + float32_blob_bytes = len(_FLOAT32_BLOB_PREFIX) + dim * numpy.dtype(numpy.float32).itemsize + float64_bytes = dim * numpy.dtype(numpy.float64).itemsize + if float32_blob_bytes == float64_bytes: + # dim=1: sizes collide, write legacy float64 to stay unambiguous + return numpy.asarray(phases, dtype=numpy.float64).tobytes() + payload = numpy.asarray(phases, dtype=numpy.float32).tobytes() + return _FLOAT32_BLOB_PREFIX + payload + + +def bytes_to_phases(data: bytes, dim: int | None = None) -> "np.ndarray": + """Deserialize a phase vector from new float32 or legacy float64 storage. + + New float32 blobs carry a small prefix so callers can round-trip without + knowing ``dim``. Legacy float64 blobs are raw NumPy bytes and remain + readable for backward compatibility. The returned array is copied and + promoted to float64 so downstream HRR math keeps the existing numerical + behavior. + + When ``dim`` is 1 the prefixed float32 blob and the raw float64 blob are + both 8 bytes, so size alone cannot disambiguate. ``phases_to_bytes`` + avoids writing prefixed blobs in that case; here we guard the remaining + collision window (a legacy float64 blob that happens to start with the + ``HRR1`` prefix) by preferring the legacy interpretation when sizes + match and the caller supplied ``dim``. + """ + numpy = _np() + + if dim is not None: + float32_payload_bytes = dim * numpy.dtype(numpy.float32).itemsize + float32_blob_bytes = len(_FLOAT32_BLOB_PREFIX) + float32_payload_bytes + float64_bytes = dim * numpy.dtype(numpy.float64).itemsize + + # When sizes collide (dim=1), prefer legacy float64 for a blob that + # starts with the prefix, because phases_to_bytes never writes a + # prefixed float32 blob at dim=1 — any such blob must be legacy. + if float32_blob_bytes == float64_bytes: + if len(data) == float64_bytes: + return numpy.frombuffer(data, dtype=numpy.float64).copy() + if data.startswith(_FLOAT32_BLOB_PREFIX): + payload_len = len(data) - len(_FLOAT32_BLOB_PREFIX) + raise ValueError( + f"HRR vector blob has {len(data)} bytes ({payload_len} payload bytes after " + f"the float32 prefix); expected {float64_bytes} (legacy float64) for dim={dim}" + ) + raise ValueError( + f"HRR legacy vector blob has {len(data)} bytes; expected " + f"{float64_bytes} (float64) for dim={dim}" + ) + + if data.startswith(_FLOAT32_BLOB_PREFIX) and len(data) == float32_blob_bytes: + payload = data[len(_FLOAT32_BLOB_PREFIX):] + return numpy.frombuffer(payload, dtype=numpy.float32).astype(numpy.float64) + if len(data) == float64_bytes: + return numpy.frombuffer(data, dtype=numpy.float64).copy() + if data.startswith(_FLOAT32_BLOB_PREFIX): + payload_len = len(data) - len(_FLOAT32_BLOB_PREFIX) + raise ValueError( + f"HRR vector blob has {len(data)} bytes ({payload_len} payload bytes after " + f"the float32 prefix); expected {float32_blob_bytes} (prefixed float32) " + f"or {float64_bytes} (legacy float64) for dim={dim}" + ) + raise ValueError( + f"HRR legacy vector blob has {len(data)} bytes; expected " + f"{float64_bytes} (float64) for dim={dim}" + ) + + if data.startswith(_FLOAT32_BLOB_PREFIX): + payload = data[len(_FLOAT32_BLOB_PREFIX):] + if len(payload) % numpy.dtype(numpy.float32).itemsize != 0: + raise ValueError( + f"HRR float32 vector blob has invalid payload byte length: {len(payload)}" + ) + return numpy.frombuffer(payload, dtype=numpy.float32).astype(numpy.float64) + + if len(data) % numpy.dtype(numpy.float64).itemsize != 0: + raise ValueError(f"HRR legacy vector blob has invalid byte length: {len(data)}") + return numpy.frombuffer(data, dtype=numpy.float64).copy() def snr_estimate(dim: int, n_items: int) -> float: diff --git a/plugins/memory/holographic/retrieval.py b/plugins/memory/holographic/retrieval.py index 95b8f2ae95db..2bbcb71f473e 100644 --- a/plugins/memory/holographic/retrieval.py +++ b/plugins/memory/holographic/retrieval.py @@ -91,7 +91,7 @@ def search( # HRR similarity if self.hrr_weight > 0 and fact.get("hrr_vector"): - fact_vec = hrr.bytes_to_phases(fact["hrr_vector"]) + fact_vec = hrr.bytes_to_phases(fact["hrr_vector"], dim=self.hrr_dim) if query_vec is None: query_vec = hrr.encode_text(query, self.hrr_dim) hrr_sim = (hrr.similarity(query_vec, fact_vec) + 1.0) / 2.0 # shift to [0,1] @@ -154,7 +154,7 @@ def probe( (bank_name,), ).fetchone() if bank_row: - bank_vec = hrr.bytes_to_phases(bank_row["vector"]) + bank_vec = hrr.bytes_to_phases(bank_row["vector"], dim=self.hrr_dim) extracted = hrr.unbind(bank_vec, probe_key) # Use extracted signal to score individual facts return self._score_facts_by_vector( @@ -189,7 +189,7 @@ def probe( scored = [] for row in rows: fact = dict(row) - fact_vec = hrr.bytes_to_phases(fact.pop("hrr_vector")) + fact_vec = hrr.bytes_to_phases(fact.pop("hrr_vector"), dim=self.hrr_dim) # Unbind probe key from fact to see if entity is structurally present residual = hrr.unbind(fact_vec, probe_key) # Compare residual against content signal @@ -253,7 +253,7 @@ def related( scored = [] for row in rows: fact = dict(row) - fact_vec = hrr.bytes_to_phases(fact.pop("hrr_vector")) + fact_vec = hrr.bytes_to_phases(fact.pop("hrr_vector"), dim=self.hrr_dim) # Check structural similarity: unbind entity from fact residual = hrr.unbind(fact_vec, entity_vec) @@ -334,7 +334,7 @@ def reason( scored = [] for row in rows: fact = dict(row) - fact_vec = hrr.bytes_to_phases(fact.pop("hrr_vector")) + fact_vec = hrr.bytes_to_phases(fact.pop("hrr_vector"), dim=self.hrr_dim) entity_scores = [] for probe_key in entity_residuals: @@ -431,8 +431,8 @@ def contradict( continue # Not enough entity overlap to be contradictory # Content similarity via HRR vectors - v1 = hrr.bytes_to_phases(f1["hrr_vector"]) - v2 = hrr.bytes_to_phases(f2["hrr_vector"]) + v1 = hrr.bytes_to_phases(f1["hrr_vector"], dim=self.hrr_dim) + v2 = hrr.bytes_to_phases(f2["hrr_vector"], dim=self.hrr_dim) content_sim = hrr.similarity(v1, v2) # High entity overlap + low content similarity = potential contradiction @@ -484,7 +484,7 @@ def _score_facts_by_vector( scored = [] for row in rows: fact = dict(row) - fact_vec = hrr.bytes_to_phases(fact.pop("hrr_vector")) + fact_vec = hrr.bytes_to_phases(fact.pop("hrr_vector"), dim=self.hrr_dim) sim = hrr.similarity(target_vec, fact_vec) fact["score"] = (sim + 1.0) / 2.0 * fact["trust_score"] scored.append(fact) diff --git a/plugins/memory/holographic/store.py b/plugins/memory/holographic/store.py index bf58d7a22dbc..fcb193d4da22 100644 --- a/plugins/memory/holographic/store.py +++ b/plugins/memory/holographic/store.py @@ -561,7 +561,7 @@ def _rebuild_bank(self, category: str) -> None: self._conn.commit() return - vectors = [hrr.bytes_to_phases(row["hrr_vector"]) for row in rows] + vectors = [hrr.bytes_to_phases(row["hrr_vector"], dim=self.hrr_dim) for row in rows] bank_vector = hrr.bundle(*vectors) fact_count = len(vectors) diff --git a/plugins/memory/honcho/client.py b/plugins/memory/honcho/client.py index cc1da55ffaf4..1945a38a15fe 100644 --- a/plugins/memory/honcho/client.py +++ b/plugins/memory/honcho/client.py @@ -22,6 +22,7 @@ from pathlib import Path from urllib.parse import urlparse +from agent.secret_scope import get_secret from hermes_constants import get_hermes_home from hermes_cli.profiles import _get_default_hermes_home from plugins.plugin_utils import SingletonSlot @@ -469,7 +470,7 @@ def from_env( ) -> HonchoClientConfig: """Create config from environment variables (fallback).""" resolved_host = host or resolve_active_host() - api_key = os.environ.get("HONCHO_API_KEY") + api_key = get_secret("HONCHO_API_KEY") base_url = os.environ.get("HONCHO_BASE_URL", "").strip() or None timeout = _resolve_optional_float(os.environ.get("HONCHO_TIMEOUT")) return cls( @@ -525,7 +526,7 @@ def from_global_config( api_key = ( host_block.get("apiKey") or raw.get("apiKey") - or os.environ.get("HONCHO_API_KEY") + or get_secret("HONCHO_API_KEY") ) environment = ( diff --git a/plugins/memory/mem0/__init__.py b/plugins/memory/mem0/__init__.py index b060a7bf7a8a..ea871c44cc41 100644 --- a/plugins/memory/mem0/__init__.py +++ b/plugins/memory/mem0/__init__.py @@ -41,6 +41,7 @@ from typing import Any, Dict, List from agent.memory_provider import MemoryProvider +from agent.secret_scope import get_secret from tools.registry import tool_error logger = logging.getLogger(__name__) @@ -85,7 +86,7 @@ def _load_config() -> dict: config = { "mode": os.environ.get("MEM0_MODE", "platform"), - "api_key": os.environ.get("MEM0_API_KEY", ""), + "api_key": get_secret("MEM0_API_KEY", ""), "host": os.environ.get("MEM0_HOST", ""), "agent_id": os.environ.get("MEM0_AGENT_ID", "hermes"), "oss": {}, diff --git a/plugins/memory/retaindb/__init__.py b/plugins/memory/retaindb/__init__.py index a777ccbd55ee..13ad0331501b 100644 --- a/plugins/memory/retaindb/__init__.py +++ b/plugins/memory/retaindb/__init__.py @@ -34,6 +34,7 @@ from urllib.parse import quote from agent.memory_provider import MemoryProvider +from agent.secret_scope import get_secret from agent.file_safety import raise_if_read_blocked from tools.registry import tool_error @@ -476,7 +477,7 @@ def name(self) -> str: return "retaindb" def is_available(self) -> bool: - return bool(os.environ.get("RETAINDB_API_KEY")) + return bool(get_secret("RETAINDB_API_KEY")) def get_config_schema(self) -> List[Dict[str, Any]]: return [ @@ -488,7 +489,7 @@ def get_config_schema(self) -> List[Dict[str, Any]]: # ── Lifecycle ────────────────────────────────────────────────────────── def initialize(self, session_id: str, **kwargs) -> None: - api_key = os.environ.get("RETAINDB_API_KEY", "") + api_key = get_secret("RETAINDB_API_KEY", "") or "" base_url = re.sub(r"/+$", "", os.environ.get("RETAINDB_BASE_URL", _DEFAULT_BASE_URL)) # Project resolution: RETAINDB_PROJECT > hermes- > "default" diff --git a/plugins/memory/supermemory/__init__.py b/plugins/memory/supermemory/__init__.py index 2112b5095d76..7d737fd36774 100644 --- a/plugins/memory/supermemory/__init__.py +++ b/plugins/memory/supermemory/__init__.py @@ -18,6 +18,7 @@ from typing import Any, Dict, List, Optional from agent.memory_provider import MemoryProvider +from agent.secret_scope import get_secret, is_multiplex_active from tools.registry import tool_error logger = logging.getLogger(__name__) @@ -572,7 +573,7 @@ def is_available(self) -> bool: # Docker venv the package isn't present until ensure() runs, but # ensure() only runs once the provider is loaded — which this gates. # Mirrors honcho/mem0, which check config only. No network calls. - return bool(os.environ.get("SUPERMEMORY_API_KEY", "")) + return bool(get_secret("SUPERMEMORY_API_KEY", "")) def get_config_schema(self): # Only prompt for the API key during `hermes memory setup`. @@ -595,7 +596,7 @@ def get_status_config(self, provider_config: dict) -> dict: del provider_config hermes_home = str(get_hermes_home()) - api_key = os.environ.get("SUPERMEMORY_API_KEY", "") + api_key = get_secret("SUPERMEMORY_API_KEY", "") or "" status = _probe_supermemory_connection(api_key, hermes_home) return {"summary": _format_connection_summary(status)} @@ -630,7 +631,15 @@ def post_setup(self, hermes_home: str, config: dict) -> None: # Make the freshly-entered key visible to the connection probe below. # (Checks the VALUE of SUPERMEMORY_API_KEY, not whether the key string # happens to name some unrelated env var.) - if api_key and os.environ.get("SUPERMEMORY_API_KEY") != api_key: + # Single-profile convenience only: never write a profile's key into + # the process-global environ under a multiplexed gateway — sibling + # profiles' turns (and any subprocess spawned with env=os.environ) + # would inherit it. + if ( + api_key + and not is_multiplex_active() + and os.environ.get("SUPERMEMORY_API_KEY") != api_key + ): os.environ["SUPERMEMORY_API_KEY"] = api_key status = _probe_supermemory_connection(api_key, hermes_home) @@ -647,7 +656,7 @@ def initialize(self, session_id: str, **kwargs) -> None: self._session_id = session_id self._turn_count = 0 self._config = _load_supermemory_config(self._hermes_home) - self._api_key = os.environ.get("SUPERMEMORY_API_KEY", "") + self._api_key = get_secret("SUPERMEMORY_API_KEY", "") or "" # Resolve container tag: env var > config > default. # Supports {identity} template for profile-scoped containers. diff --git a/plugins/model-providers/deepinfra/__init__.py b/plugins/model-providers/deepinfra/__init__.py index afa06548db16..57c6a9c4ce0c 100644 --- a/plugins/model-providers/deepinfra/__init__.py +++ b/plugins/model-providers/deepinfra/__init__.py @@ -29,9 +29,9 @@ def default_vision_model(self): # type: ignore[override] ``vision`` capability) so an image-gen/edit model that merely carries a ``vision`` tag can't be picked as a chat-completions vision backend. """ - import os + from agent.secret_scope import get_secret - if not (os.environ.get("DEEPINFRA_API_KEY") or "").strip(): + if not (get_secret("DEEPINFRA_API_KEY") or "").strip(): return None try: from hermes_cli.models import _fetch_deepinfra_models_by_tag diff --git a/plugins/platforms/buzz/adapter.py b/plugins/platforms/buzz/adapter.py index b07830ae9b6e..8b77ae334120 100644 --- a/plugins/platforms/buzz/adapter.py +++ b/plugins/platforms/buzz/adapter.py @@ -49,6 +49,30 @@ from typing import Any, Dict, List, Optional, Tuple from urllib.parse import urlsplit, urlunsplit +from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError +from agent.secret_scope import get_secret as _scoped_get_secret + + +def _get_scoped_secret(name, default=None): + """Scope-aware credential read with the default-profile startup fallback. + + Secondary profiles construct their adapters under a profile secret + scope -- the scope is authoritative and a scoped miss returns ``default`` + (no cross-profile borrow from ``os.environ``, which may hold another + profile's value). The DEFAULT profile's adapter constructs and sends + *unscoped* under multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash this path; there ``os.environ`` is that + profile's own value, so fall back to it. Same pattern as the Slack + ``SLACK_APP_TOKEN`` read (#59739) and + ``gateway/platforms/whatsapp_common.py::_get_wsecret``. + """ + try: + val = _scoped_get_secret(name, default) + except _UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + logger = logging.getLogger(__name__) from gateway.platforms.base import ( @@ -226,7 +250,7 @@ def _resolve_private_key(extra: Optional[dict] = None) -> str: NEVER log the return value. """ - key = os.getenv("BUZZ_PRIVATE_KEY", "").strip() + key = _get_scoped_secret("BUZZ_PRIVATE_KEY", "").strip() if key: return key configured = os.getenv("BUZZ_CREDENTIALS_FILE", "").strip() or (extra or {}).get("credentials_file", "") diff --git a/plugins/platforms/dingtalk/adapter.py b/plugins/platforms/dingtalk/adapter.py index a564c08e690a..9ea8330854ce 100644 --- a/plugins/platforms/dingtalk/adapter.py +++ b/plugins/platforms/dingtalk/adapter.py @@ -103,6 +103,30 @@ SendResult, ) +from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError +from agent.secret_scope import get_secret as _scoped_get_secret + + +def _get_scoped_secret(name, default=None): + """Scope-aware credential read with the default-profile startup fallback. + + Secondary profiles construct their adapters under a profile secret + scope -- the scope is authoritative and a scoped miss returns ``default`` + (no cross-profile borrow from ``os.environ``, which may hold another + profile's value). The DEFAULT profile's adapter constructs and sends + *unscoped* under multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash this path; there ``os.environ`` is that + profile's own value, so fall back to it. Same pattern as the Slack + ``SLACK_APP_TOKEN`` read (#59739) and + ``gateway/platforms/whatsapp_common.py::_get_wsecret``. + """ + try: + val = _scoped_get_secret(name, default) + except _UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + logger = logging.getLogger(__name__) MAX_MESSAGE_LENGTH = 20000 @@ -166,7 +190,7 @@ def check_dingtalk_requirements() -> bool: httpx = _httpx DINGTALK_STREAM_AVAILABLE = True HTTPX_AVAILABLE = True - if not os.getenv("DINGTALK_CLIENT_ID") or not os.getenv("DINGTALK_CLIENT_SECRET"): + if not os.getenv("DINGTALK_CLIENT_ID") or not _get_scoped_secret("DINGTALK_CLIENT_SECRET"): return False return True @@ -213,7 +237,7 @@ def __init__(self, config: PlatformConfig): self._client_id: str = extra.get("client_id") or os.getenv( "DINGTALK_CLIENT_ID", "" ) - self._client_secret: str = extra.get("client_secret") or os.getenv( + self._client_secret: str = extra.get("client_secret") or _get_scoped_secret( "DINGTALK_CLIENT_SECRET", "" ) @@ -1842,7 +1866,7 @@ def _is_connected(config) -> bool: extra = getattr(config, "extra", {}) or {} return bool( (extra.get("client_id") or os.getenv("DINGTALK_CLIENT_ID")) - and (extra.get("client_secret") or os.getenv("DINGTALK_CLIENT_SECRET")) + and (extra.get("client_secret") or _get_scoped_secret("DINGTALK_CLIENT_SECRET")) ) diff --git a/plugins/platforms/discord/adapter.py b/plugins/platforms/discord/adapter.py index 9ef2659fbd78..99e0f9a2f57b 100644 --- a/plugins/platforms/discord/adapter.py +++ b/plugins/platforms/discord/adapter.py @@ -838,34 +838,37 @@ def flush_pending(self) -> list: @staticmethod def pcm_to_wav(pcm_data: bytes, output_path: str, src_rate: int = 48000, src_channels: int = 2): - """Convert raw PCM to 16kHz mono WAV via ffmpeg.""" - with tempfile.NamedTemporaryFile(suffix=".pcm", delete=False) as f: - f.write(pcm_data) - pcm_path = f.name - try: - from hermes_cli._subprocess_compat import windows_hide_flags + """Convert raw PCM to 16kHz mono WAV via ffmpeg. - subprocess.run( - [ - resolve_ffmpeg_executable(), "-y", "-loglevel", "error", - "-f", "s16le", - "-ar", str(src_rate), - "-ac", str(src_channels), - "-i", pcm_path, - "-ar", "16000", - "-ac", "1", - output_path, - ], - check=True, - timeout=10, - stdin=subprocess.DEVNULL, - creationflags=windows_hide_flags(), - ) - finally: - try: - os.unlink(pcm_path) - except OSError: - pass + The PCM is fed straight to ffmpeg's stdin, which avoids staging it in a + temp file on every utterance. The WAV is still written to *output_path* + rather than captured from stdout: ffmpeg cannot seek on a pipe, so a + piped WAV carries placeholder (0xFFFFFFFF) RIFF/data sizes that make + strict readers misreport the length. + """ + from hermes_cli._subprocess_compat import windows_hide_flags + + subprocess.run( + [ + resolve_ffmpeg_executable(), "-y", "-loglevel", "error", + "-f", "s16le", + "-ar", str(src_rate), + "-ac", str(src_channels), + "-i", "pipe:0", + "-ar", "16000", + "-ac", "1", + output_path, + ], + input=pcm_data, + check=True, + timeout=10, + # Capture ffmpeg's -loglevel error output so a failure's + # CalledProcessError carries the actual message (parity with + # tools/transcription_tools' ffmpeg call sites) instead of + # "returned non-zero exit status N" with stderr detached. + stderr=subprocess.PIPE, + creationflags=windows_hide_flags(), + ) def _read_dm_role_auth_guild() -> Optional[int]: @@ -4481,7 +4484,18 @@ async def _process_voice_input(self, guild_id: int, user_id: int, pcm_data: byte transcript=transcript, ) except Exception as e: - logger.warning("Voice input processing failed: %s", e, exc_info=True) + # CalledProcessError from pcm_to_wav carries ffmpeg's captured + # stderr — surface it, or the log only says "exit status N". + _ff_err = getattr(e, "stderr", None) + if _ff_err: + if isinstance(_ff_err, bytes): + _ff_err = _ff_err.decode("utf-8", "replace") + logger.warning( + "Voice input processing failed: %s (ffmpeg: %s)", + e, _ff_err.strip(), exc_info=True, + ) + else: + logger.warning("Voice input processing failed: %s", e, exc_info=True) finally: try: os.unlink(wav_path) @@ -9486,7 +9500,14 @@ async def _standalone_send( except ImportError: return {"error": "aiohttp not installed. Run: pip install aiohttp"} - token = (getattr(pconfig, "token", None) or os.getenv("DISCORD_BOT_TOKEN", "")).strip() + token = (getattr(pconfig, "token", None) or "").strip() + if not token: + # Profile-scoped read: under multiplex the process env may hold a + # different profile's bot token, so honor the secret scope's verdict + # (scoped miss ⇒ no token; unscoped multiplex ⇒ UnscopedSecretError). + from agent.secret_scope import get_secret + + token = (get_secret("DISCORD_BOT_TOKEN", "") or "").strip() if not token: return {"error": "Discord standalone send: DISCORD_BOT_TOKEN is not set"} diff --git a/plugins/platforms/email/adapter.py b/plugins/platforms/email/adapter.py index 572b5c11455d..f224202b0c55 100644 --- a/plugins/platforms/email/adapter.py +++ b/plugins/platforms/email/adapter.py @@ -23,6 +23,10 @@ import re import smtplib import socket + +# Profile-scoped secret reader for multiplexing support (PR #50094) +from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError +from agent.secret_scope import get_secret as _scoped_get_secret import ssl import uuid from email.header import decode_header @@ -43,9 +47,50 @@ cache_image_from_bytes, ) from gateway.config import Platform, PlatformConfig -from utils import env_int, env_bool +from utils import is_truthy_value logger = logging.getLogger(__name__) + + +def _get_esecret(name: str, default: str = "") -> str: + """Scope-aware ``EMAIL_*`` read with the default-profile startup fallback. + + Secondary profiles run under ``_profile_runtime_scope`` — the scope is + authoritative and a scoped miss returns ``default`` (no cross-profile + borrow). The DEFAULT profile's adapter constructs and sends *unscoped* + under multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash its email path; there ``os.environ`` + is that profile's own value, so fall back to it. Same pattern as the + Slack ``SLACK_APP_TOKEN`` read (#59739) and the WhatsApp + ``_get_wsecret`` fix (5438e9c629). + """ + try: + val = _scoped_get_secret(name, default) + except _UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + +# Backwards-compatible alias for the name used by the original #59076 hunks. +_get_secret = _get_esecret + + +def _esecret_int(name: str, default: int) -> int: + """Scope-aware integer read (``env_int`` variant of ``_get_esecret``).""" + raw = str(_get_esecret(name, "")).strip() + if not raw: + return default + try: + return int(raw) + except (ValueError, TypeError): + return default + + +def _esecret_bool(name: str, default: bool = False) -> bool: + """Scope-aware boolean read (``env_bool`` variant of ``_get_esecret``).""" + return is_truthy_value(_get_esecret(name, ""), default=default) + + # Automated sender patterns — emails from these are silently ignored _NOREPLY_PATTERNS = ( "noreply", "no-reply", "no_reply", "donotreply", "do-not-reply", @@ -164,10 +209,10 @@ def check_email_requirements() -> bool: Treats blank/whitespace-only values as missing so an abandoned setup that left empty ``EMAIL_*`` keys in ``.env`` does not enable the platform (#40715). """ - addr = os.getenv("EMAIL_ADDRESS", "").strip() - pwd = os.getenv("EMAIL_PASSWORD", "").strip() - imap = os.getenv("EMAIL_IMAP_HOST", "").strip() - smtp = os.getenv("EMAIL_SMTP_HOST", "").strip() + addr = _get_secret("EMAIL_ADDRESS", "").strip() + pwd = _get_secret("EMAIL_PASSWORD", "").strip() + imap = _get_secret("EMAIL_IMAP_HOST", "").strip() + smtp = _get_secret("EMAIL_SMTP_HOST", "").strip() return all([addr, pwd, imap, smtp]) @@ -434,13 +479,13 @@ def __init__(self, config: PlatformConfig): # misleading ``[Errno 8] nodename nor servname`` (an unresolvable name) # instead of an obvious "host not set" error. extra = config.extra or {} - self._address = (os.getenv("EMAIL_ADDRESS", "") or extra.get("address", "")).strip() - self._password = os.getenv("EMAIL_PASSWORD", "") - self._imap_host = (os.getenv("EMAIL_IMAP_HOST", "") or extra.get("imap_host", "")).strip() - self._imap_port = env_int("EMAIL_IMAP_PORT", 993) - self._smtp_host = (os.getenv("EMAIL_SMTP_HOST", "") or extra.get("smtp_host", "")).strip() - self._smtp_port = env_int("EMAIL_SMTP_PORT", 587) - self._poll_interval = env_int("EMAIL_POLL_INTERVAL", 15) + self._address = (_get_secret("EMAIL_ADDRESS", "") or extra.get("address", "")).strip() + self._password = _get_secret("EMAIL_PASSWORD", "") + self._imap_host = (_get_secret("EMAIL_IMAP_HOST", "") or extra.get("imap_host", "")).strip() + self._imap_port = _esecret_int("EMAIL_IMAP_PORT", 993) + self._smtp_host = (_get_secret("EMAIL_SMTP_HOST", "") or extra.get("smtp_host", "")).strip() + self._smtp_port = _esecret_int("EMAIL_SMTP_PORT", 587) + self._poll_interval = _esecret_int("EMAIL_POLL_INTERVAL", 15) # Skip attachments — configured via config.yaml: # platforms: @@ -464,7 +509,7 @@ def __init__(self, config: PlatformConfig): # gate below is skipped. if "require_authenticated_sender" in extra: self._require_authenticated_sender = bool(extra["require_authenticated_sender"]) - elif env_bool("EMAIL_TRUST_FROM_HEADER", False): + elif _esecret_bool("EMAIL_TRUST_FROM_HEADER", False): self._require_authenticated_sender = False else: self._require_authenticated_sender = True @@ -473,7 +518,7 @@ def __init__(self, config: PlatformConfig): # own receiving server (defends against an injected header that sorts # first). Defaults to the From-domain of the agent's own address. self._authserv_id = ( - extra.get("authserv_id", "") or os.getenv("EMAIL_AUTHSERV_ID", "") + extra.get("authserv_id", "") or _get_secret("EMAIL_AUTHSERV_ID", "") ).strip().lower() # Track message IDs we've already processed to avoid duplicates @@ -756,7 +801,7 @@ def _allow_all_senders() -> bool: """ truthy = {"true", "1", "yes"} return ( - os.getenv("EMAIL_ALLOW_ALL_USERS", "").strip().lower() in truthy + _get_secret("EMAIL_ALLOW_ALL_USERS", "").strip().lower() in truthy or os.getenv("GATEWAY_ALLOW_ALL_USERS", "").strip().lower() in truthy ) @@ -771,7 +816,7 @@ def _allowlist_in_effect() -> bool: and the authentication gate is unnecessary. """ return bool( - os.getenv("EMAIL_ALLOWED_USERS", "").strip() + _get_secret("EMAIL_ALLOWED_USERS", "").strip() or os.getenv("GATEWAY_ALLOWED_USERS", "").strip() ) @@ -793,9 +838,9 @@ async def _dispatch_message(self, msg_data: Dict[str, Any]) -> None: # that the gateway will never authorize. Without this early guard, # a race between dispatch and authorization can result in the adapter # sending a reply even though the handler returned None. - allowed_raw = os.getenv("EMAIL_ALLOWED_USERS", "").strip() + allowed_raw = _get_secret("EMAIL_ALLOWED_USERS", "").strip() if not allowed_raw: - if os.getenv("EMAIL_ALLOW_ALL_USERS", "").strip().lower() not in {"true", "1", "yes"} and ( + if _get_secret("EMAIL_ALLOW_ALL_USERS", "").strip().lower() not in {"true", "1", "yes"} and ( os.getenv("GATEWAY_ALLOW_ALL_USERS", "").strip().lower() not in {"true", "1", "yes"} ): logger.debug( @@ -1204,11 +1249,11 @@ async def _standalone_send( from email.utils import formatdate extra = getattr(pconfig, "extra", {}) or {} - address = extra.get("address") or os.getenv("EMAIL_ADDRESS", "") - password = os.getenv("EMAIL_PASSWORD", "") - smtp_host = extra.get("smtp_host") or os.getenv("EMAIL_SMTP_HOST", "") + address = extra.get("address") or _get_secret("EMAIL_ADDRESS", "") + password = _get_secret("EMAIL_PASSWORD", "") + smtp_host = extra.get("smtp_host") or _get_secret("EMAIL_SMTP_HOST", "") try: - smtp_port = int(os.getenv("EMAIL_SMTP_PORT", "587")) + smtp_port = int(_get_secret("EMAIL_SMTP_PORT", "587") or "587") except (ValueError, TypeError): smtp_port = 587 diff --git a/plugins/platforms/feishu/adapter.py b/plugins/platforms/feishu/adapter.py index d86dc597a816..c21bb42e9011 100644 --- a/plugins/platforms/feishu/adapter.py +++ b/plugins/platforms/feishu/adapter.py @@ -145,6 +145,30 @@ from hermes_constants import get_hermes_home from utils import atomic_json_write, env_float, env_int +from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError +from agent.secret_scope import get_secret as _scoped_get_secret + + +def _get_scoped_secret(name, default=None): + """Scope-aware credential read with the default-profile startup fallback. + + Secondary profiles construct their adapters under a profile secret + scope -- the scope is authoritative and a scoped miss returns ``default`` + (no cross-profile borrow from ``os.environ``, which may hold another + profile's value). The DEFAULT profile's adapter constructs and sends + *unscoped* under multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash this path; there ``os.environ`` is that + profile's own value, so fall back to it. Same pattern as the Slack + ``SLACK_APP_TOKEN`` read (#59739) and + ``gateway/platforms/whatsapp_common.py::_get_wsecret``. + """ + try: + val = _scoped_get_secret(name, default) + except _UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- @@ -1553,14 +1577,14 @@ def _load_settings(extra: Dict[str, Any]) -> FeishuAdapterSettings: return FeishuAdapterSettings( app_id=str(extra.get("app_id") or os.getenv("FEISHU_APP_ID", "")).strip(), - app_secret=str(extra.get("app_secret") or os.getenv("FEISHU_APP_SECRET", "")).strip(), + app_secret=str(extra.get("app_secret") or _get_scoped_secret("FEISHU_APP_SECRET", "")).strip(), domain_name=str(extra.get("domain") or os.getenv("FEISHU_DOMAIN", "feishu")).strip().lower(), connection_mode=str( extra.get("connection_mode") or os.getenv("FEISHU_CONNECTION_MODE", "websocket") ).strip().lower(), - encrypt_key=str(extra.get("encrypt_key") or os.getenv("FEISHU_ENCRYPT_KEY", "")).strip(), + encrypt_key=str(extra.get("encrypt_key") or _get_scoped_secret("FEISHU_ENCRYPT_KEY", "")).strip(), verification_token=str( - extra.get("verification_token") or os.getenv("FEISHU_VERIFICATION_TOKEN", "") + extra.get("verification_token") or _get_scoped_secret("FEISHU_VERIFICATION_TOKEN", "") ).strip(), group_policy=os.getenv("FEISHU_GROUP_POLICY", "allowlist").strip().lower(), allowed_group_users=frozenset( diff --git a/plugins/platforms/homeassistant/adapter.py b/plugins/platforms/homeassistant/adapter.py index 8e042133eee8..6564460d47dd 100644 --- a/plugins/platforms/homeassistant/adapter.py +++ b/plugins/platforms/homeassistant/adapter.py @@ -36,6 +36,30 @@ SendResult, ) +from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError +from agent.secret_scope import get_secret as _scoped_get_secret + + +def _get_scoped_secret(name, default=None): + """Scope-aware credential read with the default-profile startup fallback. + + Secondary profiles construct their adapters under a profile secret + scope -- the scope is authoritative and a scoped miss returns ``default`` + (no cross-profile borrow from ``os.environ``, which may hold another + profile's value). The DEFAULT profile's adapter constructs and sends + *unscoped* under multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash this path; there ``os.environ`` is that + profile's own value, so fall back to it. Same pattern as the Slack + ``SLACK_APP_TOKEN`` read (#59739) and + ``gateway/platforms/whatsapp_common.py::_get_wsecret``. + """ + try: + val = _scoped_get_secret(name, default) + except _UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + logger = logging.getLogger(__name__) @@ -46,7 +70,7 @@ def check_ha_requirements() -> bool: def validate_ha_config(config: PlatformConfig) -> bool: """Return True when Home Assistant has enough credential config to connect.""" - token = (getattr(config, "token", None) or os.getenv("HASS_TOKEN", "")).strip() + token = (getattr(config, "token", None) or _get_scoped_secret("HASS_TOKEN", "")).strip() return bool(token) @@ -76,7 +100,7 @@ def __init__(self, config: PlatformConfig): # Configuration from extra extra = config.extra or {} - token = config.token or os.getenv("HASS_TOKEN", "") + token = config.token or _get_scoped_secret("HASS_TOKEN", "") url = extra.get("url") or os.getenv("HASS_URL", "http://homeassistant.local:8123") self._hass_url: str = url.rstrip("/") self._hass_token: str = token @@ -488,7 +512,7 @@ async def _standalone_send( extra = getattr(pconfig, "extra", {}) or {} hass_url = (extra.get("url") or os.getenv("HASS_URL", "")).rstrip("/") - token = (getattr(pconfig, "token", None) or os.getenv("HASS_TOKEN", "")).strip() + token = (getattr(pconfig, "token", None) or _get_scoped_secret("HASS_TOKEN", "")).strip() if not hass_url or not token: return { "error": ( diff --git a/plugins/platforms/irc/adapter.py b/plugins/platforms/irc/adapter.py index e78798adbe67..030300b7970b 100644 --- a/plugins/platforms/irc/adapter.py +++ b/plugins/platforms/irc/adapter.py @@ -35,6 +35,30 @@ import time from typing import Any, Dict, List, Optional +from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError +from agent.secret_scope import get_secret as _scoped_get_secret + + +def _get_scoped_secret(name, default=None): + """Scope-aware credential read with the default-profile startup fallback. + + Secondary profiles construct their adapters under a profile secret + scope -- the scope is authoritative and a scoped miss returns ``default`` + (no cross-profile borrow from ``os.environ``, which may hold another + profile's value). The DEFAULT profile's adapter constructs and sends + *unscoped* under multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash this path; there ``os.environ`` is that + profile's own value, so fall back to it. Same pattern as the Slack + ``SLACK_APP_TOKEN`` read (#59739) and + ``gateway/platforms/whatsapp_common.py::_get_wsecret``. + """ + try: + val = _scoped_get_secret(name, default) + except _UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- @@ -118,8 +142,8 @@ def __init__(self, config, **kwargs): if os.getenv("IRC_USE_TLS") else extra.get("use_tls", True) ) - self.server_password = os.getenv("IRC_SERVER_PASSWORD") or extra.get("server_password", "") - self.nickserv_password = os.getenv("IRC_NICKSERV_PASSWORD") or extra.get("nickserv_password", "") + self.server_password = _get_scoped_secret("IRC_SERVER_PASSWORD") or extra.get("server_password", "") + self.nickserv_password = _get_scoped_secret("IRC_NICKSERV_PASSWORD") or extra.get("nickserv_password", "") # Auth self.allowed_users: list = extra.get("allowed_users", []) @@ -685,10 +709,10 @@ def _env_enablement() -> dict | None: seed["use_tls"] = use_tls in {"1", "true", "yes"} # Passwords live in PlatformConfig.extra as well for back-compat with # existing config.yaml users; env-reads at construct time still win. - if os.getenv("IRC_SERVER_PASSWORD"): - seed["server_password"] = os.getenv("IRC_SERVER_PASSWORD") - if os.getenv("IRC_NICKSERV_PASSWORD"): - seed["nickserv_password"] = os.getenv("IRC_NICKSERV_PASSWORD") + if _get_scoped_secret("IRC_SERVER_PASSWORD"): + seed["server_password"] = _get_scoped_secret("IRC_SERVER_PASSWORD") + if _get_scoped_secret("IRC_NICKSERV_PASSWORD"): + seed["nickserv_password"] = _get_scoped_secret("IRC_NICKSERV_PASSWORD") # Optional home-channel (usually the same as IRC_CHANNEL, but can be a # dedicated reports channel). Defaults to IRC_CHANNEL so cron jobs # with ``deliver=irc`` have a sensible target without extra config. @@ -762,8 +786,8 @@ async def _standalone_send( else: use_tls = bool(extra.get("use_tls", True)) - server_password = os.getenv("IRC_SERVER_PASSWORD") or extra.get("server_password", "") - nickserv_password = os.getenv("IRC_NICKSERV_PASSWORD") or extra.get("nickserv_password", "") + server_password = _get_scoped_secret("IRC_SERVER_PASSWORD") or extra.get("server_password", "") + nickserv_password = _get_scoped_secret("IRC_NICKSERV_PASSWORD") or extra.get("nickserv_password", "") # Reject control characters in chat_id to block IRC command injection. raw_target = chat_id or channel diff --git a/plugins/platforms/line/adapter.py b/plugins/platforms/line/adapter.py index cf14782693d1..b3c9c6463ef1 100644 --- a/plugins/platforms/line/adapter.py +++ b/plugins/platforms/line/adapter.py @@ -80,6 +80,30 @@ from typing import Any, Dict, List, Optional, Set, Tuple from urllib.parse import quote as _urlquote +from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError +from agent.secret_scope import get_secret as _scoped_get_secret + + +def _get_scoped_secret(name, default=None): + """Scope-aware credential read with the default-profile startup fallback. + + Secondary profiles construct their adapters under a profile secret + scope -- the scope is authoritative and a scoped miss returns ``default`` + (no cross-profile borrow from ``os.environ``, which may hold another + profile's value). The DEFAULT profile's adapter constructs and sends + *unscoped* under multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash this path; there ``os.environ`` is that + profile's own value, so fall back to it. Same pattern as the Slack + ``SLACK_APP_TOKEN`` read (#59739) and + ``gateway/platforms/whatsapp_common.py::_get_wsecret``. + """ + try: + val = _scoped_get_secret(name, default) + except _UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- @@ -678,11 +702,11 @@ def __init__(self, config, **kwargs): # Credentials self.channel_access_token = ( - os.getenv("LINE_CHANNEL_ACCESS_TOKEN") + _get_scoped_secret("LINE_CHANNEL_ACCESS_TOKEN") or extra.get("channel_access_token", "") ) self.channel_secret = ( - os.getenv("LINE_CHANNEL_SECRET") + _get_scoped_secret("LINE_CHANNEL_SECRET") or extra.get("channel_secret", "") ) @@ -1561,9 +1585,9 @@ def _is_relative_to(child: Path, parent: Path) -> bool: def check_requirements() -> bool: """Plugin gate: require credentials AND aiohttp at runtime.""" - if not os.getenv("LINE_CHANNEL_ACCESS_TOKEN"): + if not _get_scoped_secret("LINE_CHANNEL_ACCESS_TOKEN"): return False - if not os.getenv("LINE_CHANNEL_SECRET"): + if not _get_scoped_secret("LINE_CHANNEL_SECRET"): return False try: import aiohttp # noqa: F401 @@ -1575,10 +1599,10 @@ def check_requirements() -> bool: def validate_config(config) -> bool: extra = getattr(config, "extra", {}) or {} has_token = bool( - os.getenv("LINE_CHANNEL_ACCESS_TOKEN") or extra.get("channel_access_token") + _get_scoped_secret("LINE_CHANNEL_ACCESS_TOKEN") or extra.get("channel_access_token") ) has_secret = bool( - os.getenv("LINE_CHANNEL_SECRET") or extra.get("channel_secret") + _get_scoped_secret("LINE_CHANNEL_SECRET") or extra.get("channel_secret") ) return has_token and has_secret @@ -1595,7 +1619,7 @@ def _env_enablement() -> Optional[Dict[str, Any]]: in ``.env`` without a ``platforms.line`` block in ``config.yaml``. Mirrors the IRC plugin's pattern. """ - if not (os.getenv("LINE_CHANNEL_ACCESS_TOKEN") and os.getenv("LINE_CHANNEL_SECRET")): + if not (_get_scoped_secret("LINE_CHANNEL_ACCESS_TOKEN") and _get_scoped_secret("LINE_CHANNEL_SECRET")): return None seeded: Dict[str, Any] = {} if os.getenv("LINE_PORT"): @@ -1635,7 +1659,7 @@ async def _standalone_send( """ extra = getattr(pconfig, "extra", {}) or {} token = ( - os.getenv("LINE_CHANNEL_ACCESS_TOKEN") + _get_scoped_secret("LINE_CHANNEL_ACCESS_TOKEN") or extra.get("channel_access_token", "") ) if not token or not chat_id: diff --git a/plugins/platforms/matrix/adapter.py b/plugins/platforms/matrix/adapter.py index 93926d21c39d..b8461ae27dae 100644 --- a/plugins/platforms/matrix/adapter.py +++ b/plugins/platforms/matrix/adapter.py @@ -874,6 +874,20 @@ def _pre_sanitize_matrix_markdown(text: str) -> str: return result +def _startup_env_secret(name: str) -> str: + """Read a Matrix credential at adapter-startup time, scope-aware. + + Slack pattern (#59739): a scoped read honors the installed profile's + secret scope verdict (scoped miss ⇒ empty, no borrowing the process + env); only an UNSCOPED read under multiplex (default-profile startup + loop) falls back to ``os.environ``, which is that profile's own value. + """ + try: + return (get_secret(name) or "").strip() + except UnscopedSecretError: + return os.getenv(name, "").strip() + + def check_matrix_requirements() -> bool: """Return True if the Matrix adapter can be used. @@ -885,8 +899,8 @@ def check_matrix_requirements() -> bool: forever and broke E2EE connect with ``No module named 'asyncpg'`` (#31116). Rebinds module-level type globals on success. """ - token = os.getenv("MATRIX_ACCESS_TOKEN", "") - password = os.getenv("MATRIX_PASSWORD", "") + token = _startup_env_secret("MATRIX_ACCESS_TOKEN") + password = _startup_env_secret("MATRIX_PASSWORD") homeserver = os.getenv("MATRIX_HOMESERVER", "") if not token and not password: @@ -1013,12 +1027,14 @@ def __init__(self, config: PlatformConfig): self._homeserver: str = ( config.extra.get("homeserver", "") or os.getenv("MATRIX_HOMESERVER", "") ).rstrip("/") - self._access_token: str = config.token or os.getenv("MATRIX_ACCESS_TOKEN", "") + self._access_token: str = config.token or _startup_env_secret( + "MATRIX_ACCESS_TOKEN" + ) self._user_id: str = config.extra.get("user_id", "") or os.getenv( "MATRIX_USER_ID", "" ) - self._password: str = config.extra.get("password", "") or os.getenv( - "MATRIX_PASSWORD", "" + self._password: str = config.extra.get("password", "") or _startup_env_secret( + "MATRIX_PASSWORD" ) self._e2ee_mode: str = _resolve_e2ee_mode(config.extra) self._encryption: bool = self._e2ee_mode != "off" @@ -4801,7 +4817,10 @@ async def _standalone_send( return {"error": "aiohttp not installed. Run: pip install aiohttp"} try: homeserver = (extra.get("homeserver") or os.getenv("MATRIX_HOMESERVER", "")).rstrip("/") - token = token or os.getenv("MATRIX_ACCESS_TOKEN", "") + # In-turn read: standalone sends run inside an installed secret + # scope, so honor get_secret's verdict directly (no env fallback on + # a scoped miss). + token = token or get_secret("MATRIX_ACCESS_TOKEN", "") or "" if not homeserver or not token: return {"error": "Matrix not configured (MATRIX_HOMESERVER, MATRIX_ACCESS_TOKEN required)"} txn_id = f"hermes_{int(time.time() * 1000)}_{os.urandom(4).hex()}" diff --git a/plugins/platforms/mattermost/adapter.py b/plugins/platforms/mattermost/adapter.py index 63e7cf266f8e..c1239b37def5 100644 --- a/plugins/platforms/mattermost/adapter.py +++ b/plugins/platforms/mattermost/adapter.py @@ -30,6 +30,30 @@ SendResult, ) +from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError +from agent.secret_scope import get_secret as _scoped_get_secret + + +def _get_scoped_secret(name, default=None): + """Scope-aware credential read with the default-profile startup fallback. + + Secondary profiles construct their adapters under a profile secret + scope -- the scope is authoritative and a scoped miss returns ``default`` + (no cross-profile borrow from ``os.environ``, which may hold another + profile's value). The DEFAULT profile's adapter constructs and sends + *unscoped* under multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash this path; there ``os.environ`` is that + profile's own value, so fall back to it. Same pattern as the Slack + ``SLACK_APP_TOKEN`` read (#59739) and + ``gateway/platforms/whatsapp_common.py::_get_wsecret``. + """ + try: + val = _scoped_get_secret(name, default) + except _UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + logger = logging.getLogger(__name__) # Mattermost post size limit (server default is 16383, but 4000 is the @@ -75,7 +99,7 @@ def check_mattermost_requirements() -> bool: def validate_mattermost_config(config: PlatformConfig) -> bool: """Return True when Mattermost has enough config to connect.""" extra = getattr(config, "extra", {}) or {} - token = (getattr(config, "token", None) or os.getenv("MATTERMOST_TOKEN", "")).strip() + token = (getattr(config, "token", None) or _get_scoped_secret("MATTERMOST_TOKEN", "")).strip() url = (extra.get("url", "") or os.getenv("MATTERMOST_URL", "")).strip() if not token: logger.debug("Mattermost: MATTERMOST_TOKEN not set") @@ -98,7 +122,7 @@ def __init__(self, config: PlatformConfig): config.extra.get("url", "") or os.getenv("MATTERMOST_URL", "") ).rstrip("/") - self._token: str = config.token or os.getenv("MATTERMOST_TOKEN", "") + self._token: str = config.token or _get_scoped_secret("MATTERMOST_TOKEN", "") self._bot_user_id: str = "" self._bot_username: str = "" @@ -1022,7 +1046,7 @@ async def _standalone_send( (getattr(pconfig, "extra", {}) or {}).get("url") or os.getenv("MATTERMOST_URL", "") ).rstrip("/") - token = (getattr(pconfig, "token", None) or os.getenv("MATTERMOST_TOKEN", "")).strip() + token = (getattr(pconfig, "token", None) or _get_scoped_secret("MATTERMOST_TOKEN", "")).strip() if not base_url or not token: return { "error": ( diff --git a/plugins/platforms/ntfy/adapter.py b/plugins/platforms/ntfy/adapter.py index 88741aa62f5c..935610c32046 100644 --- a/plugins/platforms/ntfy/adapter.py +++ b/plugins/platforms/ntfy/adapter.py @@ -68,6 +68,30 @@ SendResult, ) +from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError +from agent.secret_scope import get_secret as _scoped_get_secret + + +def _get_scoped_secret(name, default=None): + """Scope-aware credential read with the default-profile startup fallback. + + Secondary profiles construct their adapters under a profile secret + scope -- the scope is authoritative and a scoped miss returns ``default`` + (no cross-profile borrow from ``os.environ``, which may hold another + profile's value). The DEFAULT profile's adapter constructs and sends + *unscoped* under multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash this path; there ``os.environ`` is that + profile's own value, so fall back to it. Same pattern as the Slack + ``SLACK_APP_TOKEN`` read (#59739) and + ``gateway/platforms/whatsapp_common.py::_get_wsecret``. + """ + try: + val = _scoped_get_secret(name, default) + except _UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + logger = logging.getLogger(__name__) @@ -173,7 +197,7 @@ def __init__(self, config: PlatformConfig): or os.getenv("NTFY_PUBLISH_TOPIC", "") or self._topic ) - self._token: str = extra.get("token") or os.getenv("NTFY_TOKEN", "") + self._token: str = extra.get("token") or _get_scoped_secret("NTFY_TOKEN", "") self._stream_task: Optional[asyncio.Task] = None self._http_client: Optional["httpx.AsyncClient"] = None @@ -472,7 +496,7 @@ def _env_enablement() -> dict | None: publish_topic = os.getenv("NTFY_PUBLISH_TOPIC", "").strip() if publish_topic: seed["publish_topic"] = publish_topic - token = os.getenv("NTFY_TOKEN", "").strip() + token = _get_scoped_secret("NTFY_TOKEN", "").strip() if token: seed["token"] = token markdown = os.getenv("NTFY_MARKDOWN", "").strip().lower() @@ -526,7 +550,7 @@ async def _standalone_send( if not publish_topic: return {"error": "ntfy standalone send: NTFY_TOPIC not configured"} - token = extra.get("token") or os.getenv("NTFY_TOKEN", "") + token = extra.get("token") or _get_scoped_secret("NTFY_TOKEN", "") markdown_env = os.getenv("NTFY_MARKDOWN", "").strip().lower() markdown_enabled = bool(extra.get("markdown")) or markdown_env in ("1", "true", "yes") diff --git a/plugins/platforms/photon/adapter.py b/plugins/platforms/photon/adapter.py index abebe3669f98..d7f89c4a6925 100644 --- a/plugins/platforms/photon/adapter.py +++ b/plugins/platforms/photon/adapter.py @@ -67,6 +67,30 @@ from .auth import load_project_credentials +from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError +from agent.secret_scope import get_secret as _scoped_get_secret + + +def _get_scoped_secret(name, default=None): + """Scope-aware credential read with the default-profile startup fallback. + + Secondary profiles construct their adapters under a profile secret + scope -- the scope is authoritative and a scoped miss returns ``default`` + (no cross-profile borrow from ``os.environ``, which may hold another + profile's value). The DEFAULT profile's adapter constructs and sends + *unscoped* under multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash this path; there ``os.environ`` is that + profile's own value, so fall back to it. Same pattern as the Slack + ``SLACK_APP_TOKEN`` read (#59739) and + ``gateway/platforms/whatsapp_common.py::_get_wsecret``. + """ + try: + val = _scoped_get_secret(name, default) + except _UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- @@ -511,7 +535,7 @@ def _reinstall_sidecar_deps() -> None: def validate_config(cfg: PlatformConfig) -> bool: extra = cfg.extra or {} project_id = extra.get("project_id") or os.getenv("PHOTON_PROJECT_ID") - project_secret = extra.get("project_secret") or os.getenv("PHOTON_PROJECT_SECRET") + project_secret = extra.get("project_secret") or _get_scoped_secret("PHOTON_PROJECT_SECRET") if not project_id or not project_secret: # Fall back to auth.json stored_id, stored_sec = load_project_credentials() @@ -694,7 +718,7 @@ def __init__(self, config: PlatformConfig): or "" ) self._project_secret: str = ( - os.getenv("PHOTON_PROJECT_SECRET") + _get_scoped_secret("PHOTON_PROJECT_SECRET") or extra.get("project_secret") or stored_sec or "" @@ -707,7 +731,7 @@ def __init__(self, config: PlatformConfig): ) self._sidecar_bind = _DEFAULT_SIDECAR_BIND self._sidecar_token = ( - os.getenv("PHOTON_SIDECAR_TOKEN") or secrets.token_hex(16) + _get_scoped_secret("PHOTON_SIDECAR_TOKEN") or secrets.token_hex(16) ) self._autostart_sidecar = str( os.getenv("PHOTON_SIDECAR_AUTOSTART", "true") @@ -2730,7 +2754,7 @@ async def _standalone_send( (pconfig.extra or {}).get("sidecar_port") or os.getenv("PHOTON_SIDECAR_PORT"), _DEFAULT_SIDECAR_PORT, ) - token = os.getenv("PHOTON_SIDECAR_TOKEN") + token = _get_scoped_secret("PHOTON_SIDECAR_TOKEN") if not token: # Fall back to the runtime record the gateway persists once its # sidecar passes /healthz (issue #69960) — the token only exists in diff --git a/plugins/platforms/photon/auth.py b/plugins/platforms/photon/auth.py index b8b356a16b8c..34b573a2d810 100644 --- a/plugins/platforms/photon/auth.py +++ b/plugins/platforms/photon/auth.py @@ -53,6 +53,30 @@ except ImportError: # pragma: no cover - httpx is a hermes dependency httpx = None # type: ignore[assignment] +from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError +from agent.secret_scope import get_secret as _scoped_get_secret + + +def _get_scoped_secret(name, default=None): + """Scope-aware credential read with the default-profile startup fallback. + + Secondary profiles construct their adapters under a profile secret + scope -- the scope is authoritative and a scoped miss returns ``default`` + (no cross-profile borrow from ``os.environ``, which may hold another + profile's value). The DEFAULT profile's adapter constructs and sends + *unscoped* under multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash this path; there ``os.environ`` is that + profile's own value, so fall back to it. Same pattern as the Slack + ``SLACK_APP_TOKEN`` read (#59739) and + ``gateway/platforms/whatsapp_common.py::_get_wsecret``. + """ + try: + val = _scoped_get_secret(name, default) + except _UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + logger = logging.getLogger(__name__) @@ -231,7 +255,7 @@ def load_project_credentials() -> Tuple[Optional[str], Optional[str]]: is the unified project id (dashboard id == spectrumProjectId). """ env_id = os.getenv("PHOTON_PROJECT_ID") - env_sec = os.getenv("PHOTON_PROJECT_SECRET") + env_sec = _get_scoped_secret("PHOTON_PROJECT_SECRET") if env_id and env_sec: return env_id, env_sec auth = _load_auth() diff --git a/plugins/platforms/slack/adapter.py b/plugins/platforms/slack/adapter.py index 736598dba88f..ee50a4578ca0 100644 --- a/plugins/platforms/slack/adapter.py +++ b/plugins/platforms/slack/adapter.py @@ -8609,7 +8609,10 @@ async def _standalone_send( ``chat.postMessage``. """ del force_document # signature parity with other standalone senders - raw_token = getattr(pconfig, "token", None) or os.getenv("SLACK_BOT_TOKEN", "") + # Profile-scoped read: under multiplex os.environ may hold ANOTHER + # profile's bot token (first-writer-wins env bridges), so honor the + # secret scope's verdict instead of reading the process env directly. + raw_token = getattr(pconfig, "token", None) or get_secret("SLACK_BOT_TOKEN", "") # ``SLACK_BOT_TOKEN`` can be a comma-separated list in multi-workspace # gateways, and OAuth installs persist per-workspace tokens in diff --git a/plugins/platforms/sms/adapter.py b/plugins/platforms/sms/adapter.py index 3d794da592ae..0c081242d969 100644 --- a/plugins/platforms/sms/adapter.py +++ b/plugins/platforms/sms/adapter.py @@ -36,6 +36,30 @@ ) from gateway.platforms.helpers import redact_phone, strip_markdown +from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError +from agent.secret_scope import get_secret as _scoped_get_secret + + +def _get_scoped_secret(name, default=None): + """Scope-aware credential read with the default-profile startup fallback. + + Secondary profiles construct their adapters under a profile secret + scope -- the scope is authoritative and a scoped miss returns ``default`` + (no cross-profile borrow from ``os.environ``, which may hold another + profile's value). The DEFAULT profile's adapter constructs and sends + *unscoped* under multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash this path; there ``os.environ`` is that + profile's own value, so fall back to it. Same pattern as the Slack + ``SLACK_APP_TOKEN`` read (#59739) and + ``gateway/platforms/whatsapp_common.py::_get_wsecret``. + """ + try: + val = _scoped_get_secret(name, default) + except _UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + logger = logging.getLogger(__name__) TWILIO_API_BASE = "https://api.twilio.com/2010-04-01/Accounts" @@ -51,7 +75,7 @@ def check_sms_requirements() -> bool: import aiohttp # noqa: F401 except ImportError: return False - return bool(os.getenv("TWILIO_ACCOUNT_SID") and os.getenv("TWILIO_AUTH_TOKEN")) + return bool(_get_scoped_secret("TWILIO_ACCOUNT_SID") and _get_scoped_secret("TWILIO_AUTH_TOKEN")) class SmsAdapter(BasePlatformAdapter): @@ -66,8 +90,8 @@ class SmsAdapter(BasePlatformAdapter): def __init__(self, config: PlatformConfig): super().__init__(config, Platform.SMS) - self._account_sid: str = os.environ["TWILIO_ACCOUNT_SID"] - self._auth_token: str = os.environ["TWILIO_AUTH_TOKEN"] + self._account_sid: str = _get_scoped_secret("TWILIO_ACCOUNT_SID", "") + self._auth_token: str = _get_scoped_secret("TWILIO_AUTH_TOKEN", "") self._from_number: str = os.getenv("TWILIO_PHONE_NUMBER", "") self._webhook_port: int = int( os.getenv("SMS_WEBHOOK_PORT", str(DEFAULT_WEBHOOK_PORT)) @@ -435,14 +459,14 @@ async def _standalone_send( ): """Out-of-process SMS delivery via the Twilio REST API. Implements the standalone_sender_fn contract; replaces the legacy _send_sms helper.""" - auth_token = getattr(pconfig, "api_key", None) or os.getenv("TWILIO_AUTH_TOKEN", "") + auth_token = getattr(pconfig, "api_key", None) or _get_scoped_secret("TWILIO_AUTH_TOKEN", "") try: import aiohttp except ImportError: return {"error": "aiohttp not installed. Run: pip install aiohttp"} import base64 - account_sid = os.getenv("TWILIO_ACCOUNT_SID", "") + account_sid = _get_scoped_secret("TWILIO_ACCOUNT_SID", "") from_number = os.getenv("TWILIO_PHONE_NUMBER", "") if not account_sid or not auth_token or not from_number: return {"error": "SMS not configured (TWILIO_ACCOUNT_SID, TWILIO_AUTH_TOKEN, TWILIO_PHONE_NUMBER required)"} diff --git a/plugins/platforms/teams/adapter.py b/plugins/platforms/teams/adapter.py index d5e7035ae996..9ccbb8aaa51f 100644 --- a/plugins/platforms/teams/adapter.py +++ b/plugins/platforms/teams/adapter.py @@ -91,6 +91,30 @@ cache_media_bytes, ) +from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError +from agent.secret_scope import get_secret as _scoped_get_secret + + +def _get_scoped_secret(name, default=None): + """Scope-aware credential read with the default-profile startup fallback. + + Secondary profiles construct their adapters under a profile secret + scope -- the scope is authoritative and a scoped miss returns ``default`` + (no cross-profile borrow from ``os.environ``, which may hold another + profile's value). The DEFAULT profile's adapter constructs and sends + *unscoped* under multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash this path; there ``os.environ`` is that + profile's own value, so fall back to it. Same pattern as the Slack + ``SLACK_APP_TOKEN`` read (#59739) and + ``gateway/platforms/whatsapp_common.py::_get_wsecret``. + """ + try: + val = _scoped_get_secret(name, default) + except _UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + logger = logging.getLogger(__name__) _DEFAULT_PORT = 3978 @@ -200,7 +224,7 @@ def _resolve_delivery_config(self, config: dict[str, Any] | None) -> dict[str, A env_defaults = { "delivery_mode": os.getenv("TEAMS_DELIVERY_MODE", ""), "incoming_webhook_url": os.getenv("TEAMS_INCOMING_WEBHOOK_URL", ""), - "access_token": os.getenv("TEAMS_GRAPH_ACCESS_TOKEN", ""), + "access_token": _get_scoped_secret("TEAMS_GRAPH_ACCESS_TOKEN", ""), "team_id": os.getenv("TEAMS_TEAM_ID", ""), "channel_id": os.getenv("TEAMS_CHANNEL_ID", ""), "chat_id": os.getenv("TEAMS_CHAT_ID", ""), @@ -401,7 +425,7 @@ def validate_config(config) -> bool: """Return True when the config has the minimum required credentials.""" extra = getattr(config, "extra", {}) or {} client_id = os.getenv("TEAMS_CLIENT_ID") or extra.get("client_id", "") - client_secret = os.getenv("TEAMS_CLIENT_SECRET") or extra.get("client_secret", "") + client_secret = _get_scoped_secret("TEAMS_CLIENT_SECRET") or extra.get("client_secret", "") tenant_id = os.getenv("TEAMS_TENANT_ID") or extra.get("tenant_id", "") return bool(client_id and client_secret and tenant_id) @@ -423,7 +447,7 @@ def _env_enablement() -> dict | None: ``HomeChannel`` dataclass on the ``PlatformConfig`` via the core hook. """ client_id = os.getenv("TEAMS_CLIENT_ID", "").strip() - client_secret = os.getenv("TEAMS_CLIENT_SECRET", "").strip() + client_secret = _get_scoped_secret("TEAMS_CLIENT_SECRET", "").strip() tenant_id = os.getenv("TEAMS_TENANT_ID", "").strip() if not (client_id and client_secret and tenant_id): return None @@ -528,7 +552,7 @@ async def _standalone_send( """ extra = getattr(pconfig, "extra", {}) or {} client_id = os.getenv("TEAMS_CLIENT_ID") or extra.get("client_id", "") - client_secret = os.getenv("TEAMS_CLIENT_SECRET") or extra.get("client_secret", "") + client_secret = _get_scoped_secret("TEAMS_CLIENT_SECRET") or extra.get("client_secret", "") tenant_id = os.getenv("TEAMS_TENANT_ID") or extra.get("tenant_id", "") if not (client_id and client_secret and tenant_id): return {"error": "Teams standalone send: TEAMS_CLIENT_ID, TEAMS_CLIENT_SECRET, and TEAMS_TENANT_ID are all required"} @@ -728,7 +752,7 @@ def __init__(self, config: PlatformConfig): super().__init__(config, Platform("teams")) extra = config.extra or {} self._client_id = extra.get("client_id") or os.getenv("TEAMS_CLIENT_ID", "") - self._client_secret = extra.get("client_secret") or os.getenv("TEAMS_CLIENT_SECRET", "") + self._client_secret = extra.get("client_secret") or _get_scoped_secret("TEAMS_CLIENT_SECRET", "") self._tenant_id = extra.get("tenant_id") or os.getenv("TEAMS_TENANT_ID", "") self._port = _coerce_port( extra.get("port") or os.getenv("TEAMS_PORT", str(_DEFAULT_PORT)) diff --git a/plugins/platforms/telegram/adapter.py b/plugins/platforms/telegram/adapter.py index ef31587d223d..674a49d2be94 100644 --- a/plugins/platforms/telegram/adapter.py +++ b/plugins/platforms/telegram/adapter.py @@ -4038,7 +4038,20 @@ def _with_limits(httpx_kwargs: Optional[dict] = None) -> dict: os.getenv("TELEGRAM_WEBHOOK_HOST", "").strip() or str((self.config.extra or {}).get("webhook_host") or "").strip() ) - webhook_secret = os.getenv("TELEGRAM_WEBHOOK_SECRET", "").strip() + # Profile-scoped read (adapter startup, Slack pattern + # #59739): a scoped read honors the profile's own secret; + # only an UNSCOPED read under multiplex (default-profile + # startup loop) falls back to the process env, which is that + # profile's own value. + from agent.secret_scope import ( + UnscopedSecretError, + get_secret, + ) + + try: + webhook_secret = (get_secret("TELEGRAM_WEBHOOK_SECRET") or "").strip() + except UnscopedSecretError: + webhook_secret = os.getenv("TELEGRAM_WEBHOOK_SECRET", "").strip() if not webhook_secret: raise RuntimeError( "TELEGRAM_WEBHOOK_SECRET is required when " @@ -9932,7 +9945,13 @@ async def _standalone_send( parse-mode fallback). Implements the standalone_sender_fn contract so deliver=telegram cron jobs succeed when cron runs separately from the gateway.""" - token = getattr(pconfig, "token", None) or os.getenv("TELEGRAM_BOT_TOKEN", "") + token = getattr(pconfig, "token", None) + if not token: + # Profile-scoped read: honor the secret scope's verdict rather than + # borrowing another profile's env-bridged token under multiplex. + from agent.secret_scope import get_secret + + token = get_secret("TELEGRAM_BOT_TOKEN", "") or "" disable_link_previews = bool( getattr(pconfig, "extra", {}) and pconfig.extra.get("disable_link_previews") ) diff --git a/plugins/platforms/wecom/adapter.py b/plugins/platforms/wecom/adapter.py index 2f8e6d66fc31..715445bd236b 100644 --- a/plugins/platforms/wecom/adapter.py +++ b/plugins/platforms/wecom/adapter.py @@ -70,6 +70,30 @@ ) from utils import env_float +from agent.secret_scope import UnscopedSecretError as _UnscopedSecretError +from agent.secret_scope import get_secret as _scoped_get_secret + + +def _get_scoped_secret(name, default=None): + """Scope-aware credential read with the default-profile startup fallback. + + Secondary profiles construct their adapters under a profile secret + scope -- the scope is authoritative and a scoped miss returns ``default`` + (no cross-profile borrow from ``os.environ``, which may hold another + profile's value). The DEFAULT profile's adapter constructs and sends + *unscoped* under multiplexing, where a bare ``get_secret`` would raise + ``UnscopedSecretError`` and crash this path; there ``os.environ`` is that + profile's own value, so fall back to it. Same pattern as the Slack + ``SLACK_APP_TOKEN`` read (#59739) and + ``gateway/platforms/whatsapp_common.py::_get_wsecret``. + """ + try: + val = _scoped_get_secret(name, default) + except _UnscopedSecretError: + val = os.getenv(name) + return val if val is not None else default + + logger = logging.getLogger(__name__) DEFAULT_WS_URL = "wss://openws.work.weixin.qq.com" @@ -154,7 +178,7 @@ def __init__(self, config: PlatformConfig): extra = config.extra or {} self._bot_id = str(extra.get("bot_id") or os.getenv("WECOM_BOT_ID", "")).strip() - self._secret = str(extra.get("secret") or os.getenv("WECOM_SECRET", "")).strip() + self._secret = str(extra.get("secret") or _get_scoped_secret("WECOM_SECRET", "")).strip() self._ws_url = str( extra.get("websocket_url") or extra.get("websocketUrl") diff --git a/plugins/teams_pipeline/pipeline.py b/plugins/teams_pipeline/pipeline.py index a4c600b11ec3..38d66717bf3c 100644 --- a/plugins/teams_pipeline/pipeline.py +++ b/plugins/teams_pipeline/pipeline.py @@ -15,6 +15,8 @@ import httpx +from agent.secret_scope import get_secret + from agent.auxiliary_client import async_call_llm, extract_content_or_reasoning from hermes_constants import get_hermes_home from plugins.teams_pipeline.meetings import ( @@ -109,7 +111,7 @@ class NotionWriter: API_VERSION = "2025-09-03" def __init__(self, *, api_key: str | None = None, transport: httpx.AsyncBaseTransport | None = None) -> None: - self.api_key = (api_key or os.getenv("NOTION_API_KEY", "")).strip() + self.api_key = (api_key or get_secret("NOTION_API_KEY", "") or "").strip() self._transport = transport async def write_summary( @@ -205,7 +207,7 @@ class LinearWriter: API_URL = "https://api.linear.app/graphql" def __init__(self, *, api_key: str | None = None, transport: httpx.AsyncBaseTransport | None = None) -> None: - self.api_key = (api_key or os.getenv("LINEAR_API_KEY", "")).strip() + self.api_key = (api_key or get_secret("LINEAR_API_KEY", "") or "").strip() self._transport = transport async def write_summary( diff --git a/run_agent.py b/run_agent.py index 9a6542925989..0376c4b16cd5 100644 --- a/run_agent.py +++ b/run_agent.py @@ -115,6 +115,7 @@ def _session_source_for_agent(platform: Optional[str]) -> str: _get_proxy_for_base_url, ) from agent.iteration_budget import IterationBudget +from agent.interrupt_compat import request_hard_interrupt from hermes_cli.env_loader import load_hermes_dotenv @@ -2976,7 +2977,7 @@ def _save_session_log(self, messages: List[Dict[str, Any]] = None): logging.warning(f"Failed to save session log: {e}") - def interrupt(self, message: str = None) -> None: + def interrupt(self, message: Optional[str] = None, *, hard_cancel: bool = False) -> None: """ Request the agent to interrupt its current tool-calling loop. @@ -2989,6 +2990,9 @@ def interrupt(self, message: str = None) -> None: Args: message: Optional new message that triggered the interrupt. If provided, the agent will include this in its response context. + hard_cancel: Mark this as an explicit stop rather than a redirect or + incoming-message interrupt. Compression may honor this + atomic signal even while ordinary interrupts are masked. Example (CLI): # In a separate input thread: @@ -3002,15 +3006,41 @@ def interrupt(self, message: str = None) -> None: """ # A hard stop and redirect share one lock so /stop cannot race with an # accepted correction and accidentally turn itself into a retry. + def _admit_hard_cancel() -> None: + event = getattr(self, "_hard_interrupt_requested", None) + if event is None: + return + fence = vars(self).get("_active_compression_commit_fence") + cancel_before_commit = getattr( + type(fence), "cancel_before_commit", None + ) + if callable(cancel_before_commit): + try: + # This sets the Event while holding the same lock used by + # begin_commit(). If commit already won, it waits for that + # tracked mutation to finish before publishing the stop. + cancel_before_commit(fence, event) + return + except Exception: + logger.debug( + "Compression hard-cancel fence admission failed", + exc_info=True, + ) + event.set() + _redirect_lock = getattr(self, "_pending_redirect_lock", None) if _redirect_lock is not None: with _redirect_lock: self._interrupt_requested = True self._interrupt_message = message + if hard_cancel: + _admit_hard_cancel() self._pending_redirect = None else: self._interrupt_requested = True self._interrupt_message = message + if hard_cancel: + _admit_hard_cancel() self._pending_redirect = None # Codex app-server owns its model/tool loop and watches a private @@ -3073,12 +3103,26 @@ def interrupt(self, message: str = None) -> None: children_copy = list(self._active_children) for child in children_copy: try: - child.interrupt(message) + if hard_cancel: + request_hard_interrupt(child, message) + else: + child.interrupt(message) except Exception as e: logger.debug("Failed to propagate interrupt to child agent: %s", e) if not self.quiet_mode: print("\n⚡ Interrupt requested" + (f": '{message[:40]}...'" if message and len(message) > 40 else f": '{message}'" if message else "")) + def hard_interrupt(self, message: Optional[str] = None) -> None: + """Request an explicit stop while preserving ``interrupt()`` ABI. + + Frontends can feature-detect this method and fall back to the legacy + ``interrupt()`` signature for synthetic or third-party agents. + """ + # Deliberately bypass dynamic dispatch: subclasses written against the + # legacy interrupt(message=None) ABI may override interrupt without the + # newer keyword-only hard_cancel argument. + AIAgent.interrupt(self, message, hard_cancel=True) + def clear_interrupt(self, *, preserve_redirect: bool = False) -> bool: """Clear the interrupt request and per-thread tool signal. @@ -3093,6 +3137,7 @@ def clear_interrupt(self, *, preserve_redirect: bool = False) -> bool: return False self._interrupt_requested = False self._interrupt_message = None + getattr(self, "_hard_interrupt_requested", threading.Event()).clear() if not preserve_redirect: self._pending_redirect = None else: @@ -3100,6 +3145,7 @@ def clear_interrupt(self, *, preserve_redirect: bool = False) -> bool: return False self._interrupt_requested = False self._interrupt_message = None + getattr(self, "_hard_interrupt_requested", threading.Event()).clear() if not preserve_redirect: self._pending_redirect = None self._interrupt_thread_signal_pending = False @@ -4085,6 +4131,15 @@ def close(self) -> None: except Exception: pass + # The references above are now gone; on Linux/glibc, return their free + # heap pages immediately instead of retaining the process RSS high-water + # mark until exit. This helper is a safe no-op on other allocators. + try: + from hermes_cli.mem_trim import trim_memory + trim_memory(force=True, reason="agent close") + except Exception: + pass + # 8. Finalize the owned SQLite session row unless this agent is only a # temporary helper that deliberately handed session ownership forward # (manual compression helpers that rotate to a continuation session_id, @@ -6907,7 +6962,10 @@ def _compress_context( auto-compress abort. Auto-compress callers use the default ``force=False``. """ - from agent.conversation_compression import compress_context + from agent.conversation_compression import ( + CompressionCommitFence, + compress_context, + ) from agent.portal_tags import ( get_conversation_context, reset_conversation_context, @@ -6932,19 +6990,39 @@ def _compress_context( root = self._conversation_root_id() if root: token = set_conversation_context(root) - try: - return compress_context( - self, messages, system_message, - approx_tokens=approx_tokens, task_id=task_id, focus_topic=focus_topic, - force=force, - defer_context_engine_notification=defer_context_engine_notification, - commit_fence=commit_fence, + # Every AIAgent compression has a fence, including ordinary in-turn and + # manual paths. hard_interrupt() uses this exact instance to serialize + # cancel admission against begin_commit(). + active_fence = commit_fence or CompressionCommitFence() + # A single agent can receive overlapping automatic/manual entrypoints. + # Serialize fence publication so a waiter cannot replace the fence of + # the attempt currently generating/committing a summary. + fence_registration_lock = vars(self).setdefault( + "_compression_commit_fence_lock", threading.RLock() + ) + with fence_registration_lock: + missing_fence = object() + previous_fence = vars(self).get( + "_active_compression_commit_fence", missing_fence ) - finally: - # Restore whatever the caller had, so a compaction never leaks its - # tag into the surrounding scope. - if token is not None: - reset_conversation_context(token) + self._active_compression_commit_fence = active_fence + try: + return compress_context( + self, messages, system_message, + approx_tokens=approx_tokens, task_id=task_id, focus_topic=focus_topic, + force=force, + defer_context_engine_notification=defer_context_engine_notification, + commit_fence=active_fence, + ) + finally: + if previous_fence is missing_fence: + vars(self).pop("_active_compression_commit_fence", None) + else: + self._active_compression_commit_fence = previous_fence + # Restore whatever the caller had, so a compaction never leaks its + # tag into the surrounding scope. + if token is not None: + reset_conversation_context(token) def _set_tool_guardrail_halt(self, decision: ToolGuardrailDecision) -> None: """Record the first guardrail decision that should stop this turn.""" diff --git a/tests/agent/test_auxiliary_explicit_cancellation.py b/tests/agent/test_auxiliary_explicit_cancellation.py new file mode 100644 index 000000000000..991738af8aab --- /dev/null +++ b/tests/agent/test_auxiliary_explicit_cancellation.py @@ -0,0 +1,622 @@ +"""Deterministic cross-thread cancellation tests for compression aux transports.""" + +from __future__ import annotations + +import contextvars +import threading +import time +from types import SimpleNamespace +from typing import Any, Callable + +import pytest + +from agent import auxiliary_client as aux + + +class _BlockingStream: + def __init__(self, started: threading.Event) -> None: + self.started = started + self.closed = threading.Event() + + def __iter__(self): + self.started.set() + self.closed.wait(timeout=5) + raise RuntimeError("transport closed") + + def close(self) -> None: + self.closed.set() + + def get_final_message(self) -> Any: + self.started.set() + self.closed.wait(timeout=5) + raise RuntimeError("transport closed") + + +class _GenericCompletions: + def __init__(self, stream: _BlockingStream) -> None: + self.stream = stream + + def create(self, **_kwargs: Any) -> _BlockingStream: + return self.stream + + +class _GenericClient: + def __init__(self, stream: _BlockingStream) -> None: + self.chat = SimpleNamespace(completions=_GenericCompletions(stream)) + self.stream = stream + self.closed = threading.Event() + + def close(self) -> None: + self.closed.set() + self.stream.close() + + +class _CodexResponses: + def __init__(self, stream: _BlockingStream) -> None: + self.stream = stream + + def create(self, **_kwargs: Any) -> _BlockingStream: + return self.stream + + +class _CodexRealClient: + def __init__(self, stream: _BlockingStream) -> None: + self.responses = _CodexResponses(stream) + self.api_key = "test" + self.base_url = "https://example.test/codex" + self.stream = stream + self.closed = threading.Event() + + def close(self) -> None: + self.closed.set() + self.stream.close() + + +class _AnthropicStreamContext: + def __init__(self, stream: _BlockingStream) -> None: + self.stream = stream + + def __enter__(self) -> _BlockingStream: + return self.stream + + def __exit__(self, *_args: Any) -> None: + self.stream.close() + + +class _AnthropicMessages: + def __init__(self, stream: _BlockingStream) -> None: + self.stream_obj = stream + + def stream(self, **_kwargs: Any) -> _AnthropicStreamContext: + return _AnthropicStreamContext(self.stream_obj) + + +class _AnthropicRealClient: + def __init__(self, stream: _BlockingStream) -> None: + self.messages = _AnthropicMessages(stream) + self.stream = stream + self.closed = threading.Event() + + def close(self) -> None: + self.closed.set() + self.stream.close() + + +class _BedrockRuntimeClient: + def __init__(self, started: threading.Event, release: threading.Event) -> None: + self.started = started + self.release = release + self.closed = threading.Event() + + def converse(self, **_kwargs: Any) -> dict[str, Any]: + self.started.set() + self.release.wait(timeout=5) + return { + "output": { + "message": { + "role": "assistant", + "content": [{"text": "cancelled response"}], + } + }, + "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, + "stopReason": "end_turn", + } + + def close(self) -> None: + self.closed.set() + + +def _cancel_silent_request( + client: Any, + started: threading.Event, + invoke: Callable[[Any], Any], +) -> tuple[BaseException, float]: + cancel_event = threading.Event() + result: dict[str, BaseException] = {} + + def _worker() -> None: + try: + with aux.aux_interrupt_protection(cancel_event=cancel_event): + invoke(client) + except BaseException as exc: + result["exc"] = exc + + worker = threading.Thread(target=_worker, daemon=True) + worker.start() + assert started.wait(timeout=1), "request never entered its silent transport" + cancelled_at = time.monotonic() + cancel_event.set() + worker.join(timeout=1) + elapsed = time.monotonic() - cancelled_at + assert not worker.is_alive(), "explicit cancellation did not wake the silent request" + return result["exc"], elapsed + + +def _invoke_generic(client: Any) -> Any: + return aux._relay_sync_completion( + client, + {"model": "test", "messages": [], "timeout": 30}, + create=lambda request: aux._create_with_progress( + client, request, "compression", force_stream=True + ), + ) + + +def test_protected_silent_provider_is_isolated_and_raises_frozen_explicit_cancel() -> None: + started = threading.Event() + stream = _BlockingStream(started) + client = _GenericClient(stream) + + exc, elapsed = _cancel_silent_request(client, started, _invoke_generic) + + assert isinstance(exc, aux.AuxiliaryExplicitCancellation) + assert exc.cause == "explicit_host_cancel" + assert not client.closed.is_set() + assert elapsed < 0.75 + stream.close() # release the bounded daemon provider worker + + +def test_codex_silent_stream_is_isolated_without_closing_shared_client() -> None: + started = threading.Event() + stream = _BlockingStream(started) + real_client = _CodexRealClient(stream) + client = aux.CodexAuxiliaryClient(real_client, "gpt-test") + + exc, elapsed = _cancel_silent_request(client, started, _invoke_generic) + + assert isinstance(exc, aux.AuxiliaryExplicitCancellation) + assert not real_client.closed.is_set() + assert elapsed < 0.75 + stream.close() + + +def test_cancelled_codex_orphan_timeout_preserves_cached_shared_client() -> None: + """A cancelled Codex worker's delayed timer owns only its event stream.""" + owner_started = threading.Event() + + class _SilentOwnerStream: + def __init__(self) -> None: + self.closed = threading.Event() + + def __iter__(self): + owner_started.set() + self.closed.wait(timeout=5) + raise RuntimeError("owner stream closed") + + def close(self) -> None: + self.closed.set() + + class _SuccessStream: + def __iter__(self): + message = SimpleNamespace( + type="message", + content=[SimpleNamespace(type="output_text", text="ok")], + ) + return iter( + [ + SimpleNamespace(type="response.output_item.done", item=message), + SimpleNamespace( + type="response.completed", + response=SimpleNamespace( + status="completed", id="success", usage=None + ), + ), + ] + ) + + def close(self) -> None: + pass + + owner_stream = _SilentOwnerStream() + + class _SharedResponses: + def __init__(self, real_client: Any) -> None: + self.real_client = real_client + + def create(self, **kwargs: Any) -> Any: + if self.real_client.closed.is_set(): + raise RuntimeError("shared client was closed") + if kwargs["model"] == "owner": + return owner_stream + return _SuccessStream() + + class _SharedRealClient: + def __init__(self) -> None: + self.closed = threading.Event() + self.api_key = "test" + self.base_url = "https://example.test/codex" + self.responses = _SharedResponses(self) + + def close(self) -> None: + self.closed.set() + owner_stream.close() + + real_client = _SharedRealClient() + wrapper = aux.CodexAuxiliaryClient(real_client, "gpt-test") + cache_key = ("openai-codex", False, None, None, None) + cancel_event = threading.Event() + owner_outcome: dict[str, BaseException] = {} + + def _run_owner() -> None: + try: + with aux.aux_interrupt_protection(cancel_event=cancel_event): + aux._relay_sync_completion( + wrapper, + {"model": "owner", "messages": [], "timeout": 0.12}, + ) + except BaseException as exc: + owner_outcome["exc"] = exc + + with aux._client_cache_lock: + aux._client_cache.clear() + aux._client_cache[cache_key] = (wrapper, "gpt-test", None) + owner = threading.Thread(target=_run_owner, daemon=True) + try: + owner.start() + assert owner_started.wait(timeout=1) + cancel_event.set() + owner.join(timeout=1) + assert not owner.is_alive() + assert isinstance(owner_outcome["exc"], aux.AuxiliaryExplicitCancellation) + # A real frontend clears the reusable host Event when the next turn + # starts. The orphan must retain a frozen per-attempt cancellation cause. + cancel_event.clear() + + # A second user can use the shared client while the cancelled provider + # worker is still orphaned and its total-timeout timer is still armed. + assert not owner_stream.closed.is_set() + concurrent = aux._relay_sync_completion( + wrapper, + {"model": "concurrent", "messages": [], "timeout": 1}, + ) + assert concurrent.choices[0].message.content == "ok" + + # Let the orphan's real adapter timer fire. It may close the attempt's + # event stream to wake that worker, but never the process-shared client. + assert owner_stream.closed.wait(timeout=1) + time.sleep(0.03) + assert not real_client.closed.is_set() + with aux._client_cache_lock: + assert aux._client_cache[cache_key][0] is wrapper + + successive = aux._relay_sync_completion( + wrapper, + {"model": "successive", "messages": [], "timeout": 1}, + ) + assert successive.choices[0].message.content == "ok" + finally: + owner_stream.close() + with aux._client_cache_lock: + aux._client_cache.clear() + + +@pytest.mark.parametrize("winner", ["timeout", "cancel"]) +def test_codex_timeout_and_explicit_cancel_have_one_linearized_outcome( + winner: str, +) -> None: + """Timeout and explicit cancel can never produce a mixed owner/cleanup result.""" + timer_read_started = threading.Event() + allow_timer_read_return = threading.Event() + request_cancelled = threading.Event() + stream_started = threading.Event() + + class _RacingCancelSource: + def is_set(self) -> bool: + if winner == "timeout" and threading.current_thread().name.startswith( + "Thread-" + ): + # Take the timer's false snapshot, then hold it at the exact seam + # where the historical implementation could race owner polling. + was_set = request_cancelled.is_set() + timer_read_started.set() + assert allow_timer_read_return.wait(timeout=1) + return was_set + return request_cancelled.is_set() + + class _SilentStream: + def __init__(self) -> None: + self.closed = threading.Event() + + def __iter__(self): + stream_started.set() + self.closed.wait(timeout=5) + raise RuntimeError("stream closed") + + def close(self) -> None: + self.closed.set() + + stream = _SilentStream() + + class _RealClient: + def __init__(self) -> None: + self.api_key = "test" + self.base_url = "https://example.test/codex" + self.responses = SimpleNamespace(create=lambda **_kwargs: stream) + self.closed = threading.Event() + + def close(self) -> None: + self.closed.set() + stream.close() + + real_client: Any = _RealClient() + wrapper = aux.CodexAuxiliaryClient(real_client, "gpt-test") + owner_outcome: dict[str, BaseException] = {} + + def _run_owner() -> None: + try: + with aux.aux_interrupt_protection(cancel_event=_RacingCancelSource()): + aux._relay_sync_completion( + wrapper, + {"model": "owner", "messages": [], "timeout": 0.08}, + ) + except BaseException as exc: + owner_outcome["exc"] = exc + + owner = threading.Thread(target=_run_owner, name="race-owner", daemon=True) + owner.start() + assert stream_started.wait(timeout=1) + if winner == "timeout": + assert timer_read_started.wait(timeout=1) + request_cancelled.set() + allow_timer_read_return.set() + else: + request_cancelled.set() + owner.join(timeout=1) + + assert not owner.is_alive() + if winner == "timeout": + assert real_client.closed.is_set() + assert isinstance(owner_outcome["exc"], TimeoutError) + assert not isinstance(owner_outcome["exc"], aux.AuxiliaryExplicitCancellation) + else: + assert isinstance(owner_outcome["exc"], aux.AuxiliaryExplicitCancellation) + assert stream.closed.wait(timeout=1), "cancelled timer did not wake its stream" + assert not real_client.closed.is_set() + + +def test_anthropic_silent_stream_is_isolated_without_closing_shared_client() -> None: + started = threading.Event() + stream = _BlockingStream(started) + real_client = _AnthropicRealClient(stream) + client = aux.AnthropicAuxiliaryClient( + real_client, + "claude-test", + "test-key", + "https://api.anthropic.test", + ) + + exc, elapsed = _cancel_silent_request(client, started, _invoke_generic) + + assert isinstance(exc, aux.AuxiliaryExplicitCancellation) + assert not real_client.closed.is_set() + assert elapsed < 0.75 + stream.close() + + +def test_cancelled_attempt_does_not_close_or_fail_concurrent_shared_client_call( + monkeypatch, +) -> None: + a_started = threading.Event() + a_release = threading.Event() + b_started = threading.Event() + b_release = threading.Event() + closed = threading.Event() + + class _SharedCompletions: + def create(self, **kwargs: Any) -> Any: + if kwargs["model"] == "session-a": + a_started.set() + a_release.wait(timeout=5) + else: + b_started.set() + b_release.wait(timeout=5) + if closed.is_set(): + raise RuntimeError("shared client was closed") + return SimpleNamespace( + choices=[SimpleNamespace(message=SimpleNamespace(content="ok"))] + ) + + client = SimpleNamespace( + chat=SimpleNamespace(completions=_SharedCompletions()), + close=lambda: closed.set(), + ) + cancel_event = threading.Event() + outcomes: dict[str, Any] = {} + evictions: list[Any] = [] + monkeypatch.setattr( + aux, "_evict_cached_client_instance", lambda value: evictions.append(value) + ) + + def _session_a() -> None: + try: + with aux.aux_interrupt_protection(cancel_event=cancel_event): + aux._relay_sync_completion( + client, {"model": "session-a", "messages": [], "timeout": 30} + ) + except BaseException as exc: + outcomes["a"] = exc + + def _session_b() -> None: + try: + outcomes["b"] = aux._relay_sync_completion( + client, {"model": "session-b", "messages": [], "timeout": 30} + ) + except BaseException as exc: # pragma: no cover - asserted below + outcomes["b"] = exc + + a_thread = threading.Thread(target=_session_a, daemon=True) + b_thread = threading.Thread(target=_session_b, daemon=True) + a_thread.start() + b_thread.start() + assert a_started.wait(timeout=1) + assert b_started.wait(timeout=1) + cancel_event.set() + a_thread.join(timeout=1) + try: + assert not a_thread.is_alive() + assert isinstance(outcomes["a"], aux.AuxiliaryExplicitCancellation) + assert not closed.is_set() + assert evictions == [] + b_release.set() + b_thread.join(timeout=1) + assert not b_thread.is_alive() + assert not isinstance(outcomes["b"], BaseException) + assert outcomes["b"].choices[0].message.content == "ok" + finally: + a_release.set() + b_release.set() + + +def test_bedrock_silent_nonstream_request_is_isolated_without_close_wakeup() -> None: + from agent.bedrock_adapter import _bedrock_runtime_client_cache, reset_client_cache + + started = threading.Event() + release = threading.Event() + runtime_client = _BedrockRuntimeClient(started, release) + reset_client_cache() + _bedrock_runtime_client_cache["us-test-1"] = runtime_client + client = aux.BedrockAuxiliaryClient("us-test-1", "bedrock-test") + try: + exc, elapsed = _cancel_silent_request(client, started, _invoke_generic) + finally: + release.set() + reset_client_cache() + + assert isinstance(exc, aux.AuxiliaryExplicitCancellation) + assert not runtime_client.closed.is_set() + assert elapsed < 0.75 + + +def test_unprotected_sync_completion_stays_on_calling_thread() -> None: + caller = threading.get_ident() + observed: list[int] = [] + client = SimpleNamespace( + chat=SimpleNamespace( + completions=SimpleNamespace( + create=lambda **_kwargs: ( + observed.append(threading.get_ident()), + SimpleNamespace(choices=[]), + )[1] + ) + ) + ) + + aux._relay_sync_completion(client, {"model": "test", "messages": []}) + + assert observed == [caller] + + +def test_isolated_provider_worker_inherits_protection_and_progress_hook() -> None: + caller = threading.get_ident() + cancel_event = threading.Event() + progress: list[str] = [] + observed: dict[str, Any] = {} + + def _create(**_kwargs: Any) -> Any: + observed["thread"] = threading.get_ident() + observed["protected"] = aux._aux_interrupt_protected() + aux._notify_aux_progress() + return SimpleNamespace(choices=[]) + + client = SimpleNamespace( + chat=SimpleNamespace(completions=SimpleNamespace(create=_create)) + ) + with aux.aux_progress_hook(lambda: progress.append("tick")), aux.aux_interrupt_protection( + cancel_event=cancel_event + ): + aux._relay_sync_completion(client, {"model": "test", "messages": []}) + + assert observed["protected"] is True + assert observed["thread"] != caller + assert progress == ["tick"] + + +def test_isolated_provider_worker_inherits_caller_contextvars() -> None: + from tools.approval import ( + get_current_session_key, + reset_current_session_key, + set_current_session_key, + ) + + arbitrary = contextvars.ContextVar("isolated-provider-test", default="missing") + arbitrary_token = arbitrary.set("caller-value") + session_token = set_current_session_key("session-from-caller") + observed: dict[str, str] = {} + client = SimpleNamespace( + chat=SimpleNamespace( + completions=SimpleNamespace( + create=lambda **_kwargs: ( + observed.update( + arbitrary=arbitrary.get(), + session_key=get_current_session_key(), + ), + SimpleNamespace(choices=[]), + )[1] + ) + ) + ) + try: + with aux.aux_interrupt_protection(cancel_event=threading.Event()): + aux._relay_sync_completion(client, {"model": "test", "messages": []}) + finally: + reset_current_session_key(session_token) + arbitrary.reset(arbitrary_token) + + assert observed == { + "arbitrary": "caller-value", + "session_key": "session-from-caller", + } + + +def test_hard_cancel_wins_when_provider_result_is_published_in_same_race() -> None: + cancel_event = threading.Event() + + def _create(**_kwargs: Any) -> Any: + cancel_event.set() + return SimpleNamespace(choices=[]) + + client = SimpleNamespace( + chat=SimpleNamespace(completions=SimpleNamespace(create=_create)) + ) + with aux.aux_interrupt_protection(cancel_event=cancel_event): + with pytest.raises(aux.AuxiliaryExplicitCancellation): + aux._relay_sync_completion(client, {"model": "test", "messages": []}) + + +def test_unrelated_interrupted_error_is_not_reclassified_as_explicit_cancel() -> None: + client = SimpleNamespace( + chat=SimpleNamespace( + completions=SimpleNamespace( + create=lambda **_kwargs: (_ for _ in ()).throw( + InterruptedError("provider syscall interrupted") + ) + ) + ), + close=lambda: None, + ) + + with aux.aux_interrupt_protection(cancel_event=threading.Event()): + with pytest.raises(InterruptedError, match="provider syscall interrupted") as caught: + aux._relay_sync_completion(client, {"model": "test", "messages": []}) + + assert not isinstance(caught.value, aux.AuxiliaryExplicitCancellation) diff --git a/tests/agent/test_compression_concurrent_fork.py b/tests/agent/test_compression_concurrent_fork.py index 7afa6d6ddcea..7dc164c425d0 100644 --- a/tests/agent/test_compression_concurrent_fork.py +++ b/tests/agent/test_compression_concurrent_fork.py @@ -28,8 +28,10 @@ from __future__ import annotations +import copy import inspect import os +import sqlite3 import threading import time from pathlib import Path @@ -40,7 +42,12 @@ from hermes_state import SessionDB -def _build_agent_with_db(db: SessionDB, session_id: str): +def _build_agent_with_db( + db: SessionDB, + session_id: str, + *, + stub_compressor: bool = True, +): """Build an AIAgent that's wired to ``db`` and pinned to ``session_id``.""" with patch.dict(os.environ, {"OPENROUTER_API_KEY": "test-key"}): from run_agent import AIAgent @@ -60,6 +67,9 @@ def _build_agent_with_db(db: SessionDB, session_id: str): # an LLM call. Sleep inside compress() so the two threads' rotations # actually overlap — without that the OS could happen to serialize them # and hide the bug. + if not stub_compressor: + return agent + compressor = MagicMock() def _compress_with_overlap(*_a, **_kw): @@ -871,7 +881,504 @@ def test_lease_refresher_failure_window_is_bounded_by_ttl() -> None: ) +def test_hard_interrupt_aborts_compression_and_unblocks_session_writes(tmp_path: Path) -> None: + """Ctrl+C must abort an interrupt-protected summary without leaving the + session write-blocked behind its compression lease.""" + from agent import auxiliary_client as aux + + db = SessionDB(db_path=tmp_path / "state.db") + session_id = "HARD_INTERRUPT_COMPRESSION_TEST" + db.create_session(session_id, source="cli") + + agent = _build_agent_with_db(db, session_id) + messages = [{"role": "user", "content": f"m{i}"} for i in range(20)] + original_messages = copy.deepcopy(messages) + + def _cancelled_compress(*_args, **_kwargs): + agent._hard_interrupt_requested.set() + assert aux._aux_interrupt_cancel_requested() is True + messages[0]["content"] = "must be rolled back" + raise aux.AuxiliaryExplicitCancellation() + + agent.context_compressor.compress.side_effect = _cancelled_compress + + compressed, _prompt = agent._compress_context( + messages, "sys", approx_tokens=120_000 + ) + + assert compressed == original_messages + assert messages == original_messages + assert db.get_compression_lock_holder(session_id) is None + db.append_message(session_id, "assistant", "writes recovered") + + +def test_late_hard_interrupt_restores_full_compressor_attempt_state_and_retry( + tmp_path: Path, +) -> None: + """A stop after provider success but before compress() returns is a true no-op.""" + db = SessionDB(db_path=tmp_path / "state.db") + session_id = "LATE_HARD_INTERRUPT_STATE_TEST" + db.create_session(session_id, source="cli") + agent = _build_agent_with_db(db, session_id) + agent.compression_in_place = True + agent._cached_system_prompt = "sys" + messages = [{"role": "user", "content": f"m{i}"} for i in range(20)] + provider_returned = threading.Event() + allow_compress_return = threading.Event() + shared_telemetry = {"shared": [1, 2, 3]} + state_fields = { + "_previous_summary": "old-summary", + "_summary_has_user_turn": False, + "compression_count": 4, + "_last_compression_savings_pct": 37.5, + "_ineffective_compression_count": 1, + "_anti_thrash_recovery_deadline": 123.0, + "_fallback_compression_streak": 1, + "_verify_compaction_cleared_threshold": False, + "_last_compression_made_progress": False, + "_summary_failure_cooldown_until": 456.0, + "_cooldown_persist_failed": True, + "_last_summary_error": "old-error", + "_consecutive_timeout_failures": 2, + "_last_summary_dropped_count": 3, + "_last_summary_fallback_used": True, + "_last_compress_aborted": False, + "_last_summary_auth_failure": True, + "_last_summary_network_failure": True, + "_last_aux_model_failure_error": "old-aux-error", + "_last_aux_model_failure_model": "old-aux-model", + "_summary_model_fallen_back": True, + "summary_model": "old-summary-model", + "_last_compression_telemetry": shared_telemetry, + "_active_compression_telemetry": shared_telemetry, + "_compression_telemetry_seed": {"seed": [3]}, + } + for name, value in state_fields.items(): + setattr(agent.context_compressor, name, copy.deepcopy(value)) + restored_shared_telemetry = copy.deepcopy(shared_telemetry) + agent.context_compressor._last_compression_telemetry = restored_shared_telemetry + agent.context_compressor._active_compression_telemetry = restored_shared_telemetry + + def _provider_succeeded_then_waits(*_args, **_kwargs): + for name in state_fields: + setattr(agent.context_compressor, name, f"mutated-{name}") + provider_returned.set() + assert allow_compress_return.wait(timeout=5) + return [ + {"role": "user", "content": "[CONTEXT COMPACTION] cancelled summary"}, + {"role": "user", "content": "tail"}, + ] + + agent.context_compressor.compress.side_effect = _provider_succeeded_then_waits + result: dict[str, tuple] = {} + worker = threading.Thread( + target=lambda: result.setdefault( + "value", agent._compress_context(messages, "sys", approx_tokens=120_000) + ), + daemon=True, + ) + worker.start() + assert provider_returned.wait(timeout=2) + agent.hard_interrupt("cancel after provider return") + allow_compress_return.set() + worker.join(timeout=5) + + assert not worker.is_alive() + assert result["value"][0] is messages + assert { + name: copy.deepcopy(getattr(agent.context_compressor, name)) + for name in state_fields + } == state_fields + assert ( + agent.context_compressor._active_compression_telemetry + is agent.context_compressor._last_compression_telemetry + ) + assert db.get_compression_lock_holder(session_id) is None + + agent.clear_interrupt() + agent.context_compressor.compress.side_effect = lambda *_a, **_kw: [ + {"role": "user", "content": "[CONTEXT COMPACTION] retry summary"}, + {"role": "user", "content": "tail"}, + ] + retried, _prompt = agent._compress_context( + messages, "sys", approx_tokens=120_000 + ) + assert retried is not messages + assert retried[0]["content"] == "[CONTEXT COMPACTION] retry summary" + + +def test_force_cancel_restores_newer_durable_cooldown_captured_under_lease( + tmp_path: Path, +) -> None: + """A stale forced attempt rolls back to the lease-protected durable row.""" + from agent.auxiliary_client import AuxiliaryExplicitCancellation + from agent.context_compressor import ContextCompressor + + db = SessionDB(db_path=tmp_path / "state.db") + session_id = "FORCE_CANCEL_DURABLE_COOLDOWN" + db.create_session(session_id, source="cli") + + # B binds first and therefore has no local cooldown. A then persists a + # newer cooldown for the same durable session before B acquires its lease. + stale_agent = _build_agent_with_db( + db, session_id, stub_compressor=False + ) + writer_agent = _build_agent_with_db( + db, session_id, stub_compressor=False + ) + stale = stale_agent.context_compressor + writer = writer_agent.context_compressor + assert isinstance(stale, ContextCompressor) + assert isinstance(writer, ContextCompressor) + assert stale._summary_failure_cooldown_until == 0.0 + + writer._record_compression_failure_cooldown(120.0, "newer durable failure") + durable_before = tuple( + db._conn.execute( + "SELECT compression_failure_cooldown_until, compression_failure_error " + "FROM sessions WHERE id = ?", + (session_id,), + ).fetchone() + ) + assert durable_before[0] is not None + + stale_seed = {"seed": ["truly-pre-attempt"]} + stale._compression_telemetry_seed = copy.deepcopy(stale_seed) + stale._previous_summary = "pre-attempt-summary" + stale_agent._compression_feasibility_checked = True + stale_agent.compression_in_place = True + messages = [{"role": "user", "content": f"m{i}"} for i in range(20)] + + real_clear = ContextCompressor._clear_compression_failure_cooldown + + def _clear_then_hard_cancel() -> None: + real_clear(stale) + stale_agent._hard_interrupt_requested.set() + raise AuxiliaryExplicitCancellation() + + # Exercise the built-in force=True mutation point deterministically: force + # clears the durable cooldown, then the frozen host cancellation unwinds it. + stale._clear_compression_failure_cooldown = _clear_then_hard_cancel + + compressed, _prompt = stale_agent._compress_context( + messages, + "sys", + approx_tokens=120_000, + force=True, + ) + + assert compressed is messages + durable_after = tuple( + db._conn.execute( + "SELECT compression_failure_cooldown_until, compression_failure_error " + "FROM sessions WHERE id = ?", + (session_id,), + ).fetchone() + ) + assert durable_after == durable_before + assert stale._summary_failure_cooldown_until > time.monotonic() + assert stale._last_summary_error == "newer durable failure" + assert stale._cooldown_persist_failed is False + assert stale._compression_telemetry_seed == stale_seed + assert stale._previous_summary == "pre-attempt-summary" + assert db.get_compression_lock_holder(session_id) is None + + # A future compressor refresh must still observe the exact row rather than + # the cancelled force attempt having permanently cleared it. + future_agent = _build_agent_with_db( + db, session_id, stub_compressor=False + ) + future = future_agent.context_compressor.get_active_compression_failure_cooldown( + refresh=True + ) + assert future is not None + assert future["error"] == "newer durable failure" + + +def test_unrelated_interrupted_error_propagates_and_releases_compression_lease( + tmp_path: Path, +) -> None: + """A plugin/OS InterruptedError is a failure, not an explicit transaction abort.""" + db = SessionDB(db_path=tmp_path / "state.db") + session_id = "UNRELATED_INTERRUPT_COMPRESSION_TEST" + db.create_session(session_id, source="cli") + agent = _build_agent_with_db(db, session_id) + messages = [{"role": "user", "content": f"m{i}"} for i in range(20)] + + def _provider_interrupted(*_args, **_kwargs): + messages[0]["content"] = "must be rolled back" + raise InterruptedError("provider syscall interrupted") + agent.context_compressor.compress.side_effect = _provider_interrupted + with pytest.raises(InterruptedError, match="provider syscall interrupted"): + agent._compress_context(messages, "sys", approx_tokens=120_000) + + assert db.get_compression_lock_holder(session_id) is None + db.append_message(session_id, "assistant", "writes recovered") + + +def test_redirect_interrupt_remains_protected_during_compression(tmp_path: Path) -> None: + """Redirects use interrupt_requested=True/message=None; only the atomic + hard-cancel event may override summary protection.""" + from agent import auxiliary_client as aux + + db = SessionDB(db_path=tmp_path / "state.db") + session_id = "REDIRECT_COMPRESSION_TEST" + db.create_session(session_id, source="cli") + agent = _build_agent_with_db(db, session_id) + agent._interrupt_requested = True + agent._interrupt_message = None + agent._pending_redirect = "new correction" + messages = [{"role": "user", "content": f"m{i}"} for i in range(20)] + + def _protected_noop(current, **_kwargs): + assert aux._aux_interrupt_cancel_requested() is False + return copy.deepcopy(current) + + agent.context_compressor.compress.side_effect = _protected_noop + + compressed, _prompt = agent._compress_context( + messages, "sys", approx_tokens=120_000 + ) + assert compressed == messages + assert db.get_compression_lock_holder(session_id) is None + + +def test_hard_cancel_between_compress_return_and_commit_begin_wins_atomically( + tmp_path: Path, +) -> None: + """The hard-stop admission and commit admission share one fence lock.""" + db = SessionDB(db_path=tmp_path / "state.db") + session_id = "HARD_CANCEL_COMMIT_RACE" + db.create_session(session_id, source="tui") + agent = _build_agent_with_db(db, session_id) + agent.compression_in_place = True + agent._cached_system_prompt = "sys" + messages = [{"role": "user", "content": f"m{i}"} for i in range(20)] + before_commit = threading.Event() + allow_commit_check = threading.Event() + + class _CommitBarrierList(list): + def __eq__(self, other): + before_commit.set() + assert allow_commit_check.wait(timeout=5) + return super().__eq__(other) + + agent.context_compressor.compress.side_effect = lambda *_a, **_kw: _CommitBarrierList( + [ + {"role": "user", "content": "[CONTEXT COMPACTION] summary"}, + {"role": "user", "content": "tail"}, + ] + ) + archive_spy = MagicMock(wraps=db.archive_and_compact) + db.archive_and_compact = archive_spy + result: dict[str, tuple] = {} + worker = threading.Thread( + target=lambda: result.setdefault( + "value", agent._compress_context(messages, "sys", approx_tokens=120_000) + ), + daemon=True, + ) + worker.start() + assert before_commit.wait(timeout=2) + + agent.hard_interrupt("cancel before commit admission") + allow_commit_check.set() + worker.join(timeout=5) + + assert not worker.is_alive() + assert result["value"][0] is messages + archive_spy.assert_not_called() + assert db.get_compression_lock_holder(session_id) is None + + +def test_hard_stop_waits_for_commit_already_admitted(tmp_path: Path) -> None: + """A surfaced stop never races an untracked post-return transcript commit.""" + db = SessionDB(db_path=tmp_path / "state.db") + session_id = "HARD_CANCEL_AFTER_COMMIT_ADMISSION" + db.create_session(session_id, source="tui") + agent = _build_agent_with_db(db, session_id) + agent.compression_in_place = True + agent._cached_system_prompt = "sys" + messages = [{"role": "user", "content": f"m{i}"} for i in range(20)] + commit_started = threading.Event() + allow_commit = threading.Event() + stop_returned = threading.Event() + original_archive = db.archive_and_compact + + def _blocked_archive(*args, **kwargs): + commit_started.set() + assert allow_commit.wait(timeout=5) + return original_archive(*args, **kwargs) + + db.archive_and_compact = _blocked_archive + agent.context_compressor.compress.side_effect = lambda *_a, **_kw: [ + {"role": "user", "content": "[CONTEXT COMPACTION] summary"}, + {"role": "user", "content": "tail"}, + ] + compression_result: dict[str, tuple] = {} + compression = threading.Thread( + target=lambda: compression_result.setdefault( + "value", agent._compress_context(messages, "sys", approx_tokens=120_000) + ), + daemon=True, + ) + compression.start() + assert commit_started.wait(timeout=2) + + stop = threading.Thread( + target=lambda: ( + agent.hard_interrupt("stop after commit admission"), + stop_returned.set(), + ), + daemon=True, + ) + stop.start() + assert not stop_returned.wait(timeout=0.1) + allow_commit.set() + compression.join(timeout=5) + stop.join(timeout=5) + + assert not compression.is_alive() + assert not stop.is_alive() + assert stop_returned.is_set() + assert compression_result["value"][0][0]["content"] == ( + "[CONTEXT COMPACTION] summary" + ) + assert agent._hard_interrupt_requested.is_set() + assert db.get_compression_lock_holder(session_id) is None + + +@pytest.mark.parametrize("deadline_offset", [-10.0, 0.05, None]) +def test_force_cancel_restores_exact_expired_or_expiring_cooldown_row( + tmp_path: Path, + deadline_offset: float | None, +) -> None: + """Cancellation preserves raw cooldown columns even after their deadline.""" + from agent.auxiliary_client import AuxiliaryExplicitCancellation + from agent.context_compressor import ContextCompressor + + db = SessionDB(db_path=tmp_path / "state.db") + session_id = f"RAW_COOLDOWN_{deadline_offset}" + db.create_session(session_id, source="cli") + deadline = time.time() + deadline_offset if deadline_offset is not None else None + db.restore_compression_failure_cooldown_row( + session_id, + { + "session_exists": True, + "cooldown_until": deadline, + "error": "expired-but-exact", + }, + ) + before = db.get_compression_failure_cooldown_row(session_id) + + agent = _build_agent_with_db(db, session_id, stub_compressor=False) + compressor = agent.context_compressor + assert isinstance(compressor, ContextCompressor) + # A stale local persistence-failure marker must not suppress restoration + # once the raw durable row was captured authoritatively under the lease. + compressor._cooldown_persist_failed = True + agent._compression_feasibility_checked = True + agent.compression_in_place = True + messages = [{"role": "user", "content": f"m{i}"} for i in range(20)] + real_clear = ContextCompressor._clear_compression_failure_cooldown + + def _mutate_then_cancel() -> None: + real_clear(compressor) + if deadline_offset is not None and deadline_offset > 0: + assert deadline is not None + while time.time() <= deadline: + time.sleep(0.005) + agent._hard_interrupt_requested.set() + raise AuxiliaryExplicitCancellation() + + compressor._clear_compression_failure_cooldown = _mutate_then_cancel + + compressed, _prompt = agent._compress_context( + messages, + "sys", + approx_tokens=120_000, + force=True, + ) + + assert compressed is messages + assert db.get_compression_failure_cooldown_row(session_id) == before + assert db.get_compression_lock_holder(session_id) is None + + +def test_cooldown_rollback_failure_surfaces_and_releases_lease( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A failed compensating write cannot masquerade as a mutation-free cancel.""" + from agent.auxiliary_client import AuxiliaryExplicitCancellation + from agent.context_compressor import ContextCompressor + + db = SessionDB(db_path=tmp_path / "state.db") + session_id = "COOLDOWN_ROLLBACK_WRITE_FAILURE" + db.create_session(session_id, source="cli") + db.record_compression_failure_cooldown( + session_id, + time.time() + 120.0, + "must-restore", + ) + agent = _build_agent_with_db(db, session_id, stub_compressor=False) + compressor = agent.context_compressor + assert isinstance(compressor, ContextCompressor) + agent._compression_feasibility_checked = True + agent.compression_in_place = True + messages = [{"role": "user", "content": f"m{i}"} for i in range(20)] + real_clear = ContextCompressor._clear_compression_failure_cooldown + + def _mutate_then_cancel() -> None: + real_clear(compressor) + agent._hard_interrupt_requested.set() + raise AuxiliaryExplicitCancellation() + + compressor._clear_compression_failure_cooldown = _mutate_then_cancel + + def _rollback_write_fails(_self, _session_id, _snapshot) -> None: + raise sqlite3.OperationalError("forced rollback write failure") + + monkeypatch.setattr( + SessionDB, + "restore_compression_failure_cooldown_row", + _rollback_write_fails, + ) + + with pytest.raises(sqlite3.OperationalError, match="forced rollback write failure"): + agent._compress_context( + messages, + "sys", + approx_tokens=120_000, + force=True, + ) + + assert db.get_compression_lock_holder(session_id) is None + + +def test_exact_cooldown_restore_api_propagates_sqlite_write_failure( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + db = SessionDB(db_path=tmp_path / "state.db") + session_id = "RAW_COOLDOWN_WRITE_FAILURE" + db.create_session(session_id, source="test") + + def _write_fails(_callback) -> None: + raise sqlite3.OperationalError("forced low-level write failure") + + monkeypatch.setattr(db, "_execute_write", _write_fails) + + with pytest.raises(sqlite3.OperationalError, match="forced low-level write failure"): + db.restore_compression_failure_cooldown_row( + session_id, + { + "session_exists": True, + "cooldown_until": time.time() + 10.0, + "error": "must propagate", + }, + ) diff --git a/tests/agent/test_compression_interrupt_protection.py b/tests/agent/test_compression_interrupt_protection.py index 075630c108c7..7aa96f15bde7 100644 --- a/tests/agent/test_compression_interrupt_protection.py +++ b/tests/agent/test_compression_interrupt_protection.py @@ -14,6 +14,8 @@ from unittest.mock import patch +import pytest + import agent.auxiliary_client as aux @@ -38,6 +40,15 @@ def test_restores_on_exception(self): assert aux._aux_interrupt_protected() is False + def test_nested_protection_preserves_explicit_cancel_check(self): + """A hard-cancel hook installed by the compression host survives the + compressor's nested protection scope.""" + with aux.aux_interrupt_protection(cancel_check=lambda: True): + with aux.aux_interrupt_protection(): + assert aux._aux_interrupt_protected() is True + assert aux._aux_interrupt_cancel_requested() is True + assert aux._aux_interrupt_cancel_requested() is False + class TestCompressionProtectsSummaryCall: """The compressor must wrap its summary call_llm in aux_interrupt_protection @@ -82,3 +93,83 @@ def fake_call_llm(**kwargs): ) # Protection must be cleared after the call returns. assert aux._aux_interrupt_protected() is False + + def test_explicit_interrupt_is_not_degraded_into_summary_fallback(self): + """Ctrl+C cancellation must escape summary fallback so the outer + compression transaction can abort without rotating the session.""" + from agent.context_compressor import ContextCompressor + + with patch("agent.context_compressor.get_model_context_length", return_value=100000): + c = ContextCompressor(model="test", quiet_mode=True) + + msgs = [ + {"role": "user", "content": "do a thing"}, + {"role": "assistant", "content": "working"}, + {"role": "user", "content": "more"}, + {"role": "assistant", "content": "done"}, + ] + with aux.aux_interrupt_protection(cancel_check=lambda: True), patch( + "agent.context_compressor.call_llm", + side_effect=aux.AuxiliaryExplicitCancellation(), + ): + try: + c._generate_summary(msgs) + except aux.AuxiliaryExplicitCancellation as exc: + assert exc.cause == "explicit_host_cancel" + else: + raise AssertionError("compression swallowed an explicit interrupt") + + def test_non_explicit_interrupted_error_remains_provider_failure(self): + """An unrelated provider/OS InterruptedError must keep the established + summary-failure fallback semantics when no host cancel was requested.""" + from agent.context_compressor import ContextCompressor + + with patch("agent.context_compressor.get_model_context_length", return_value=100000): + c = ContextCompressor(model="test", quiet_mode=True) + + msgs = [ + {"role": "user", "content": "do a thing"}, + {"role": "assistant", "content": "working"}, + {"role": "user", "content": "more"}, + {"role": "assistant", "content": "done"}, + ] + with patch( + "agent.context_compressor.call_llm", + side_effect=InterruptedError("provider syscall interrupted"), + ): + assert c._generate_summary(msgs) is None + + def test_explicit_interrupt_restores_rehydration_state(self): + """Cancellation after the handoff scan must be a compressor no-op.""" + from agent.context_compressor import ContextCompressor + + with patch("agent.context_compressor.get_model_context_length", return_value=100000): + c = ContextCompressor( + model="test", + protect_first_n=1, + protect_last_n=1, + quiet_mode=True, + ) + c._previous_summary = "foreign-session-summary" + c._summary_has_user_turn = False + msgs = [ + {"role": "system", "content": "system"}, + {"role": "user", "content": "one"}, + {"role": "assistant", "content": "two"}, + {"role": "user", "content": "three"}, + {"role": "assistant", "content": "four"}, + {"role": "user", "content": "five"}, + {"role": "assistant", "content": "six"}, + {"role": "user", "content": "tail"}, + ] + + with patch.object( + c, + "_generate_summary", + side_effect=aux.AuxiliaryExplicitCancellation(), + ): + with pytest.raises(aux.AuxiliaryExplicitCancellation): + c.compress(msgs) + + assert c._previous_summary == "foreign-session-summary" + assert c._summary_has_user_turn is False diff --git a/tests/agent/test_interrupt_compat.py b/tests/agent/test_interrupt_compat.py new file mode 100644 index 000000000000..5b62f18e1935 --- /dev/null +++ b/tests/agent/test_interrupt_compat.py @@ -0,0 +1,103 @@ +"""Compatibility contract for explicit hard-stop producers.""" + +from __future__ import annotations + +import threading +from unittest.mock import MagicMock + +from agent.interrupt_compat import request_hard_interrupt + + +class _ModernAgent: + def __init__(self) -> None: + self.calls: list[tuple[str, str | None]] = [] + + def hard_interrupt(self, message: str | None = None) -> None: + self.calls.append(("hard", message)) + + def interrupt(self, message: str | None = None) -> None: + self.calls.append(("soft", message)) + + +class _LegacyAgent: + def __init__(self) -> None: + self.calls: list[tuple[str, str | None]] = [] + + def interrupt(self, message: str | None = None) -> None: + self.calls.append(("legacy", message)) + + +def test_explicit_producer_prefers_feature_detected_hard_interrupt() -> None: + agent = _ModernAgent() + + assert request_hard_interrupt(agent, "stop now") is True + + assert agent.calls == [("hard", "stop now")] + + +def test_explicit_producer_falls_back_to_old_interrupt_signature() -> None: + agent = _LegacyAgent() + + assert request_hard_interrupt(agent, "stop now") is True + + assert agent.calls == [("legacy", "stop now")] + + +def test_explicit_producer_reports_unsupported_agent() -> None: + assert request_hard_interrupt(object(), "stop now") is False + + +def test_dynamic_proxy_does_not_fabricate_hard_interrupt_support() -> None: + agent = MagicMock() + + assert request_hard_interrupt(agent, "stop now") is True + + agent.interrupt.assert_called_once_with("stop now") + agent.hard_interrupt.assert_not_called() + + +def test_inherited_hard_interrupt_bypasses_legacy_subclass_override() -> None: + from run_agent import AIAgent + + class LegacySubclass(AIAgent): + def __init__(self) -> None: + self.legacy_calls: list[str | None] = [] + self._hard_interrupt_requested = threading.Event() + self._pending_redirect_lock = threading.RLock() + self._pending_redirect = None + self._execution_thread_id = None + self._interrupt_thread_signal_pending = False + self._tool_worker_threads: set[int] = set() + self._tool_worker_threads_lock = threading.Lock() + self._active_children: list[object] = [] + self._active_children_lock = threading.Lock() + self.quiet_mode = True + self.api_mode = "test" + + def interrupt(self, message: str | None = None) -> None: # type: ignore[override] + self.legacy_calls.append(message) + + agent = LegacySubclass() + + assert request_hard_interrupt(agent, "stop now") is True + + assert agent.legacy_calls == [] + assert agent._hard_interrupt_requested.is_set() + assert agent._interrupt_requested is True + assert agent._interrupt_message == "stop now" + + +def test_tui_subagent_interrupt_is_an_explicit_hard_stop() -> None: + import tools.delegate_tool as delegate_tool + + agent = _ModernAgent() + subagent_id = "sa-hard-stop-test" + with delegate_tool._active_subagents_lock: + delegate_tool._active_subagents[subagent_id] = {"agent": agent} + try: + assert delegate_tool.interrupt_subagent(subagent_id) is True + finally: + with delegate_tool._active_subagents_lock: + delegate_tool._active_subagents.pop(subagent_id, None) + + assert agent.calls == [("hard", f"Interrupted via TUI ({subagent_id})")] diff --git a/tests/agent/test_model_metadata.py b/tests/agent/test_model_metadata.py index ac8ed899cc3e..874b97d5dddc 100644 --- a/tests/agent/test_model_metadata.py +++ b/tests/agent/test_model_metadata.py @@ -489,6 +489,79 @@ def test_live_codex_context_replaces_stale_cache_in_both_directions( +# ========================================================================= +# Custom endpoint model metadata +# ========================================================================= + +class TestFetchEndpointModelMetadata: + def setup_method(self): + import agent.model_metadata as mm + mm._endpoint_model_metadata_cache.clear() + mm._endpoint_model_metadata_cache_time.clear() + + @pytest.mark.parametrize("status_code", [401, 403]) + def test_auth_failure_stops_after_first_candidate(self, status_code): + import agent.model_metadata as mm + + response = MagicMock() + response.status_code = status_code + response.raise_for_status.side_effect = RuntimeError(str(status_code)) + + with patch("agent.model_metadata.requests.get", return_value=response) as mock_get: + result = mm.fetch_endpoint_model_metadata("https://custom.example/v1") + + assert result == {} + mock_get.assert_called_once() + assert mock_get.call_args.kwargs["stream"] is True + response.raise_for_status.assert_not_called() + response.json.assert_not_called() + response.close.assert_called_once() + + def test_auth_failure_empty_result_is_cached(self): + import agent.model_metadata as mm + + response = MagicMock() + response.status_code = 401 + response.raise_for_status.side_effect = RuntimeError("401") + + with patch("agent.model_metadata.requests.get", return_value=response) as mock_get: + first = mm.fetch_endpoint_model_metadata("https://custom.example/v1") + second = mm.fetch_endpoint_model_metadata("https://custom.example/v1") + + assert first == second == {} + mock_get.assert_called_once() + response.close.assert_called_once() + + def test_not_found_still_tries_alternate_candidate(self): + import agent.model_metadata as mm + + not_found = MagicMock() + not_found.status_code = 404 + not_found.raise_for_status.side_effect = RuntimeError("404") + success = MagicMock() + success.status_code = 200 + success.json.return_value = { + "data": [{"id": "test/model", "context_length": 32768}] + } + + with patch( + "agent.model_metadata.requests.get", + side_effect=[not_found, success], + ) as mock_get: + result = mm.fetch_endpoint_model_metadata("https://custom.example/v1") + + assert result["test/model"]["context_length"] == 32768 + assert mock_get.call_count == 2 + assert [call.args[0] for call in mock_get.call_args_list] == [ + "https://custom.example/v1/models", + "https://custom.example/models", + ] + assert all(call.kwargs["stream"] is True for call in mock_get.call_args_list) + not_found.json.assert_not_called() + not_found.close.assert_called_once() + success.close.assert_called_once() + + # ========================================================================= # Nous Portal context-window resolution (provider="nous") # ========================================================================= diff --git a/tests/agent/test_secret_scope.py b/tests/agent/test_secret_scope.py index 1ac078d6eb90..f0d76fe65435 100644 --- a/tests/agent/test_secret_scope.py +++ b/tests/agent/test_secret_scope.py @@ -113,8 +113,103 @@ def test_nested_scopes_restore(self): class TestEnvFileParsing: """load_env_file parses without mutating os.environ.""" + def test_load_env_file_unescapes_quoted_values(self, tmp_path): + """Values written by save_env_value must round-trip byte-exactly. + + Regression: load_env_file stripped only the outer quotes, leaving + the writer's \\" and \\\\ escapes literal — credentials containing + '\"' or '\\' worked interactively but were corrupted under scoped + (cron / multiplex) resolution. + """ + from hermes_cli.config import _quote_env_value + + original = 'tok"en\\with spaces' + (tmp_path / ".env").write_text(f"MY_TOKEN={_quote_env_value(original)}\n") + assert ss.load_env_file(tmp_path / ".env") == {"MY_TOKEN": original} + + def test_load_env_file_single_quotes_and_plain_values(self, tmp_path): + (tmp_path / ".env").write_text( + "PLAIN=abc123\nQUOTED='single quoted'\nEMPTY=\n" + ) + assert ss.load_env_file(tmp_path / ".env") == { + "PLAIN": "abc123", + "QUOTED": "single quoted", + "EMPTY": "", + } + + def test_inline_comment_stripped_from_unquoted_value(self, tmp_path): + """`KEY=value # comment` → `value` (python-dotenv semantics).""" + (tmp_path / ".env").write_text("KEY=value # comment\nTABBED=foo\t#tabbed\n") + assert ss.load_env_file(tmp_path / ".env") == { + "KEY": "value", + "TABBED": "foo", + } + + def test_hash_without_preceding_whitespace_is_not_a_comment(self, tmp_path): + """`KEY=foo#bar` stays intact — dotenv only strips `#` after whitespace.""" + (tmp_path / ".env").write_text("KEY=foo#bar\nLEAD=#leading\n") + assert ss.load_env_file(tmp_path / ".env") == { + "KEY": "foo#bar", + "LEAD": "#leading", + } + + def test_inline_comment_after_quoted_value(self, tmp_path): + """Quotes strip AND the trailing comment drops; inner `#` survives.""" + (tmp_path / ".env").write_text( + "DQ=\"has # inside\" # trailing\n" + "SQ='single # inside' # trailing\n" + ) + assert ss.load_env_file(tmp_path / ".env") == { + "DQ": "has # inside", + "SQ": "single # inside", + } + def test_inline_comment_with_escaped_quote_inside_value(self, tmp_path): + r"""Escape-aware close-quote scan: `\"` must not terminate the value.""" + (tmp_path / ".env").write_text( + 'KEY="a \\" quote # x" # trail\n' + ) + assert ss.load_env_file(tmp_path / ".env") == {"KEY": 'a " quote # x'} + + def test_round_trip_writer_value_with_trailing_comment(self, tmp_path): + """A value quoted by the save_env_value writer survives an appended + inline comment byte-exactly.""" + from hermes_cli.config import _quote_env_value + + original = 'we#ird "tok\\en" # not a comment' + quoted = _quote_env_value(original) + (tmp_path / ".env").write_text(f"MY_TOKEN={quoted} # rotated 2026-08\n") + assert ss.load_env_file(tmp_path / ".env") == {"MY_TOKEN": original} + + + + + def test_strips_utf8_bom_from_first_key(self, tmp_path): + """Windows editors often save .env as UTF-8 with BOM (EF BB BF). + + Plain utf-8 keeps U+FEFF on the first key name, so get_secret('NAME') + misses under an installed scope. utf-8-sig strips the leading BOM. + """ + env = tmp_path / ".env" + env.write_bytes( + b"\xef\xbb\xbfANTHROPIC_API_KEY=sk-x\nOPENAI_API_KEY=sk-y\n" + ) + out = ss.load_env_file(env) + assert out == { + "ANTHROPIC_API_KEY": "sk-x", + "OPENAI_API_KEY": "sk-y", + } + assert "\ufeffANTHROPIC_API_KEY" not in out + scope = ss.build_profile_secret_scope(tmp_path) + ss.set_multiplex_active(True) + token = ss.set_secret_scope(scope) + try: + assert ss.get_secret("ANTHROPIC_API_KEY") == "sk-x" + assert ss.get_secret("OPENAI_API_KEY") == "sk-y" + finally: + ss.reset_secret_scope(token) + ss.set_multiplex_active(False) def test_build_profile_secret_scope(self, tmp_path): (tmp_path / ".env").write_text("ANTHROPIC_API_KEY=sk-profile\n") @@ -155,3 +250,40 @@ def test_build_profile_secret_scope_ignores_other_home_external_secrets( ) assert ss.build_profile_secret_scope(profile) == {} + + +class TestApiServerListenerGlobals: + """API_SERVER listener settings are deployment config (#69379), not + profile secrets: the scoped runner reload must keep seeing container env + (Docker compose ``environment:`` block). API_SERVER_KEY IS a credential + and stays profile-scoped.""" + + LISTENER_VARS = ( + "API_SERVER_ENABLED", + "API_SERVER_HOST", + "API_SERVER_PORT", + "API_SERVER_CORS_ORIGINS", + ) + + def test_listener_vars_read_environ_even_when_scoped_multiplex(self, monkeypatch): + for name in self.LISTENER_VARS: + monkeypatch.setenv(name, f"container-{name.lower()}") + ss.set_multiplex_active(True) + token = ss.set_secret_scope({"TELEGRAM_BOT_TOKEN": "scoped"}) + try: + for name in self.LISTENER_VARS: + assert ss.get_secret(name) == f"container-{name.lower()}" + finally: + ss.reset_secret_scope(token) + + def test_api_server_key_stays_profile_scoped(self, monkeypatch): + monkeypatch.setenv("API_SERVER_KEY", "default-profile-key-0123456789abcdef") + ss.set_multiplex_active(True) + token = ss.set_secret_scope({"OTHER": "x"}) + try: + # A scoped miss must NOT borrow the (potentially cross-profile) + # environ value: API_SERVER_KEY is a credential. + assert ss.get_secret("API_SERVER_KEY") is None + finally: + ss.reset_secret_scope(token) + assert not ss._is_global_env("API_SERVER_KEY") diff --git a/tests/agent/test_secret_scope_tier1_migration.py b/tests/agent/test_secret_scope_tier1_migration.py new file mode 100644 index 000000000000..cceb32e21f61 --- /dev/null +++ b/tests/agent/test_secret_scope_tier1_migration.py @@ -0,0 +1,233 @@ +"""Regression tests for the Tier-1 core-gateway secret-scope migration. + +Class-closure follow-up to the profile secret-scope cluster (#76462 / +#76574): representative call sites from each migrated cluster are exercised +against the three canonical scope semantics: + +- scoped value wins (the installed profile's secret is used), +- scoped miss does NOT borrow the process env under multiplex (no-borrow), +- unscoped-under-multiplex behavior per pattern: + * in-turn sites (get_secret direct) propagate/honor UnscopedSecretError + semantics via get_secret's verdict, + * startup sites (Slack pattern) fall back to os.environ on + UnscopedSecretError. +""" + +import pytest + +from agent import secret_scope as ss + + +@pytest.fixture(autouse=True) +def _reset_multiplex(): + ss.set_multiplex_active(False) + yield + ss.set_multiplex_active(False) + + +class _Scope: + """Context manager installing a secret scope.""" + + def __init__(self, mapping): + self.mapping = mapping + self.token = None + + def __enter__(self): + self.token = ss.set_secret_scope(self.mapping) + return self + + def __exit__(self, *exc): + ss.reset_secret_scope(self.token) + + +# ── Cluster A: gateway/pairing.py allowlist reads ───────────────────────── + +class TestPairingAllowlistRead: + def test_scoped_value_wins(self, monkeypatch): + from gateway.pairing import _read_allowlist_env + + monkeypatch.setenv("TELEGRAM_ALLOWED_USERS", "111") + ss.set_multiplex_active(True) + with _Scope({"TELEGRAM_ALLOWED_USERS": "222"}): + assert _read_allowlist_env("TELEGRAM_ALLOWED_USERS") == "222" + + def test_scoped_miss_no_borrow(self, monkeypatch): + from gateway.pairing import _read_allowlist_env + + monkeypatch.setenv("TELEGRAM_ALLOWED_USERS", "other-profile") + ss.set_multiplex_active(True) + with _Scope({"UNRELATED": "x"}): + assert _read_allowlist_env("TELEGRAM_ALLOWED_USERS") == "" + + def test_unscoped_multiplex_falls_back_to_env(self, monkeypatch): + # Slack pattern: unscoped read under multiplex uses the process env. + from gateway.pairing import _read_allowlist_env + + monkeypatch.setenv("TELEGRAM_ALLOWED_USERS", "own-env") + ss.set_multiplex_active(True) + assert _read_allowlist_env("TELEGRAM_ALLOWED_USERS") == "own-env" + + +# ── Cluster A: gateway/authz_mixin.py gate reads ─────────────────────────── + +class TestAuthzPlatformGateEnv: + def test_scoped_value_wins(self, monkeypatch): + from gateway.authz_mixin import _platform_gate_env + + monkeypatch.setenv("DISCORD_ALLOW_BOTS", "none") + ss.set_multiplex_active(True) + with _Scope({"DISCORD_ALLOW_BOTS": "all"}): + assert _platform_gate_env("DISCORD_ALLOW_BOTS", "none") == "all" + + def test_scoped_miss_returns_default_not_env(self, monkeypatch): + from gateway.authz_mixin import _platform_gate_env + + monkeypatch.setenv("DISCORD_ALLOW_BOTS", "all") # another profile's bridge + ss.set_multiplex_active(True) + with _Scope({"UNRELATED": "x"}): + assert _platform_gate_env("DISCORD_ALLOW_BOTS", "none") == "none" + + def test_single_profile_legacy_env(self, monkeypatch): + from gateway.authz_mixin import _platform_gate_env + + monkeypatch.setenv("GATEWAY_ALLOWED_USERS", "42") + assert _platform_gate_env("GATEWAY_ALLOWED_USERS") == "42" + + +# ── Cluster B: matrix startup reads (Slack pattern) ──────────────────────── + +class TestMatrixStartupSecret: + def _helper(self): + mod = pytest.importorskip("plugins.platforms.matrix.adapter") + return mod._startup_env_secret + + def test_scoped_value_wins(self, monkeypatch): + helper = self._helper() + monkeypatch.setenv("MATRIX_ACCESS_TOKEN", "env-token") + ss.set_multiplex_active(True) + with _Scope({"MATRIX_ACCESS_TOKEN": "scoped-token"}): + assert helper("MATRIX_ACCESS_TOKEN") == "scoped-token" + + def test_scoped_miss_no_borrow(self, monkeypatch): + helper = self._helper() + monkeypatch.setenv("MATRIX_ACCESS_TOKEN", "other-profile") + ss.set_multiplex_active(True) + with _Scope({"UNRELATED": "x"}): + assert helper("MATRIX_ACCESS_TOKEN") == "" + + def test_unscoped_multiplex_falls_back(self, monkeypatch): + helper = self._helper() + monkeypatch.setenv("MATRIX_PASSWORD", "own-env-pass") + ss.set_multiplex_active(True) + assert helper("MATRIX_PASSWORD") == "own-env-pass" + + +# ── Cluster C: managed tool gateway token override ───────────────────────── + +class TestToolGatewayUserToken: + def test_scoped_value_wins(self, monkeypatch): + from tools.managed_tool_gateway import _read_user_token_override + + monkeypatch.setenv("TOOL_GATEWAY_USER_TOKEN", "env-tok") + ss.set_multiplex_active(True) + with _Scope({"TOOL_GATEWAY_USER_TOKEN": "scoped-tok"}): + assert _read_user_token_override() == "scoped-tok" + + def test_scoped_miss_no_borrow(self, monkeypatch): + from tools.managed_tool_gateway import _read_user_token_override + + monkeypatch.setenv("TOOL_GATEWAY_USER_TOKEN", "other-profile-tok") + ss.set_multiplex_active(True) + with _Scope({"UNRELATED": "x"}): + assert _read_user_token_override() is None + + def test_unscoped_multiplex_falls_back(self, monkeypatch): + from tools.managed_tool_gateway import _read_user_token_override + + monkeypatch.setenv("TOOL_GATEWAY_USER_TOKEN", "own-env-tok") + ss.set_multiplex_active(True) + assert _read_user_token_override() == "own-env-tok" + + +class TestOpenRouterCheckApiKey: + def test_scoped_value_wins(self, monkeypatch): + from tools.openrouter_client import check_api_key + + monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) + ss.set_multiplex_active(True) + with _Scope({"OPENROUTER_API_KEY": "sk-or-scoped"}): + assert check_api_key() is True + + def test_scoped_miss_no_borrow(self, monkeypatch): + from tools.openrouter_client import check_api_key + + monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-other-profile") + ss.set_multiplex_active(True) + with _Scope({"UNRELATED": "x"}): + assert check_api_key() is False + + +# ── Cluster D: auxiliary client key resolution ────────────────────────────── + +class TestAuxiliaryScopedKeyEnv: + def test_scoped_value_wins(self, monkeypatch): + from agent.auxiliary_client import _scoped_key_env + + monkeypatch.setenv("OPENROUTER_API_KEY", "sk-env") + ss.set_multiplex_active(True) + with _Scope({"OPENROUTER_API_KEY": "sk-scoped"}): + assert _scoped_key_env("OPENROUTER_API_KEY") == "sk-scoped" + + def test_scoped_miss_no_borrow(self, monkeypatch): + from agent.auxiliary_client import _scoped_key_env + + monkeypatch.setenv("OPENAI_API_KEY", "sk-other-profile") + ss.set_multiplex_active(True) + with _Scope({"UNRELATED": "x"}): + assert _scoped_key_env("OPENAI_API_KEY") == "" + + def test_unscoped_multiplex_falls_back(self, monkeypatch): + from agent.auxiliary_client import _scoped_key_env + + monkeypatch.setenv("OPENAI_API_KEY", "sk-own-env") + ss.set_multiplex_active(True) + assert _scoped_key_env("OPENAI_API_KEY") == "sk-own-env" + + def test_empty_name_returns_empty(self): + from agent.auxiliary_client import _scoped_key_env + + assert _scoped_key_env("") == "" + + +# ── Cluster E: azure identity presence reads ──────────────────────────────── + +class TestAzureIdentityPresence: + def _describe(self): + azure = pytest.importorskip("agent.azure_identity_adapter") + if not azure.has_azure_identity_installed(): + pytest.skip("azure-identity not installed") + return azure.describe_active_credential + + def test_scoped_client_secret_detected(self, monkeypatch): + describe = self._describe() + monkeypatch.setenv("AZURE_CLIENT_ID", "cid") + monkeypatch.setenv("AZURE_TENANT_ID", "tid") + monkeypatch.delenv("AZURE_CLIENT_SECRET", raising=False) + monkeypatch.delenv("AZURE_FEDERATED_TOKEN_FILE", raising=False) + ss.set_multiplex_active(True) + with _Scope({"AZURE_CLIENT_SECRET": "scoped-secret"}): + info = describe(timeout_seconds=0.01, allow_install=False) + assert any("EnvironmentCredential" in s for s in info.get("env_sources", [])) + + def test_scoped_miss_hides_env_secret(self, monkeypatch): + describe = self._describe() + monkeypatch.setenv("AZURE_CLIENT_ID", "cid") + monkeypatch.setenv("AZURE_TENANT_ID", "tid") + monkeypatch.setenv("AZURE_CLIENT_SECRET", "other-profile-secret") + monkeypatch.delenv("AZURE_FEDERATED_TOKEN_FILE", raising=False) + ss.set_multiplex_active(True) + with _Scope({"UNRELATED": "x"}): + info = describe(timeout_seconds=0.01, allow_install=False) + assert not any( + "EnvironmentCredential" in s for s in info.get("env_sources", []) + ) diff --git a/tests/agent/test_subagent_lifecycle.py b/tests/agent/test_subagent_lifecycle.py index ae1aa5a73fda..c32241fa8f32 100644 --- a/tests/agent/test_subagent_lifecycle.py +++ b/tests/agent/test_subagent_lifecycle.py @@ -24,9 +24,15 @@ def __init__(self, ident="sa-test"): self.provider = "test" self.model = "test-model" self.interrupted = False + self.interrupt_kind = None def interrupt(self, _reason): self.interrupted = True + self.interrupt_kind = "soft" + + def hard_interrupt(self, _reason): + self.interrupted = True + self.interrupt_kind = "hard" @pytest.fixture @@ -76,6 +82,17 @@ def test_cancel_is_cooperative_and_forged_handle_is_unknown(lifecycle): assert other_service.status(handle).state is SubagentState.UNKNOWN +def test_cancel_uses_explicit_hard_interrupt(lifecycle): + handle = lifecycle.launch(SubagentLaunchRequest(goal="x")) + record = lifecycle._record(handle) + assert record is not None and record.agent is not None + + assert lifecycle.cancel(handle, reason="explicit user cancel").accepted + + assert record.agent.interrupt_kind == "hard" + lifecycle.wait(handle, timeout_seconds=1) + + diff --git a/tests/gateway/test_64674_multiplex_primary_token_scope.py b/tests/gateway/test_64674_multiplex_primary_token_scope.py index 1ce5f91abdf2..44398aec19f6 100644 --- a/tests/gateway/test_64674_multiplex_primary_token_scope.py +++ b/tests/gateway/test_64674_multiplex_primary_token_scope.py @@ -49,6 +49,64 @@ def test_unscoped_when_multiplex_off(self, tmp_path, monkeypatch): cfg = run_mod.load_gateway_config_for_runner() assert cfg.multiplex_profiles is False + def test_scoped_reload_still_sees_container_api_server_env(self, tmp_path, monkeypatch): + """#69379 — container-env API_SERVER_* visible during the scoped reload. + + Docker/systemd deployments enable the api_server platform via the + process environment (compose ``environment:`` block), not the profile + ``.env``. The multiplex runner reload happens inside the default + profile's secret scope; the listener settings are on the global + allowlist (deployment config, not profile secrets) so they must stay + visible there — while API_SERVER_KEY (a credential) still resolves + through the profile scope. + """ + from agent import secret_scope as ss + from gateway import run as run_mod + + home = tmp_path / "home" + home.mkdir() + # Credentials belong in the profile .env; listener settings do not. + (home / ".env").write_text( + "TELEGRAM_BOT_TOKEN=default-profile-token-123\n" + "API_SERVER_KEY=profile-scoped-key-0123456789abcdef\n", + encoding="utf-8", + ) + (home / "config.yaml").write_text( + "gateway:\n multiplex_profiles: true\n", encoding="utf-8" + ) + monkeypatch.setenv("HERMES_HOME", str(home)) + # Listener settings live ONLY in os.environ — the Docker compose case. + monkeypatch.setenv("API_SERVER_ENABLED", "true") + monkeypatch.setenv("API_SERVER_HOST", "0.0.0.0") + monkeypatch.setenv("API_SERVER_PORT", "8642") + monkeypatch.delenv("API_SERVER_KEY", raising=False) + monkeypatch.delenv("TELEGRAM_BOT_TOKEN", raising=False) + monkeypatch.setattr(run_mod, "get_hermes_home", lambda: home) + monkeypatch.setattr(run_mod, "_hermes_home", home) + # Model the real multiplexed gateway: run.py flips the runtime flag + # before the runner reload, making any installed scope authoritative. + ss.set_multiplex_active(True) + + cfg = run_mod.load_gateway_config_for_runner() + + assert cfg.multiplex_profiles is True + # Telegram token from the profile scope (.env) + tg = cfg.platforms.get(Platform.TELEGRAM) + assert tg is not None + assert tg.token == "default-profile-token-123" + # api_server present: key from the profile scope, listener settings + # from the container environment via the global allowlist. + api = cfg.platforms.get(Platform.API_SERVER) + assert api is not None, ( + "api_server should be enabled from container env even inside " + "the scoped runner reload (#69379)" + ) + assert api.enabled is True + assert api.extra.get("key") == "profile-scoped-key-0123456789abcdef" + assert api.extra.get("host") == "0.0.0.0" + assert api.extra.get("port") == 8642 + + class TestPlatformHasBotCredential: def test_telegram_empty_token_false(self): diff --git a/tests/gateway/test_active_session_text_merge.py b/tests/gateway/test_active_session_text_merge.py index e3dd819b2884..ac8df747c650 100644 --- a/tests/gateway/test_active_session_text_merge.py +++ b/tests/gateway/test_active_session_text_merge.py @@ -115,6 +115,63 @@ def _debounced_event(adapter: BasePlatformAdapter, session_key: str) -> MessageE return adapter._text_debounce[session_key].event +@pytest.mark.asyncio +async def test_non_dm_message_does_not_wait_for_topic_recovery_executor(monkeypatch): + """Group messages must not queue behind the shared thread pool. + + Topic recovery only applies to Telegram DM topic mode. Offloading that + no-op check for every group message makes ingress wait behind unrelated + blocking jobs when the default executor is saturated. + """ + adapter = _make_adapter() + recovery = MagicMock(return_value=None) + adapter.set_topic_recovery_fn(recovery) + executor_called = False + never_release = asyncio.Event() + + async def _blocked_to_thread(*args, **kwargs): + nonlocal executor_called + executor_called = True + await never_release.wait() + + monkeypatch.setattr(asyncio, "to_thread", _blocked_to_thread) + + await asyncio.wait_for( + adapter.handle_message(_make_event("/status", chat_type="group")), + timeout=1.0, + ) + await asyncio.sleep(0) + + assert executor_called is False + recovery.assert_not_called() + + +@pytest.mark.asyncio +async def test_dm_topic_recovery_stays_offloaded(monkeypatch): + """Real Telegram DM topic recovery must still run outside the event loop.""" + adapter = _make_adapter() + recovery = MagicMock(return_value="topic-222") + adapter.set_topic_recovery_fn(recovery) + offloaded = False + + async def _inline_to_thread(func, *args, **kwargs): + nonlocal offloaded + offloaded = True + return func(*args, **kwargs) + + monkeypatch.setattr(asyncio, "to_thread", _inline_to_thread) + event = _make_event("hello", chat_type="dm", thread_id="1") + original_source = event.source + + await adapter.handle_message(event) + await asyncio.sleep(0) + + assert offloaded is True + assert recovery.call_count == 1 + assert recovery.call_args.args[0] is original_source + assert event.source.thread_id == "topic-222" + + @pytest.mark.asyncio async def test_rapid_text_followups_accumulate_instead_of_replacing(): """Rapid TEXT follow-ups must all survive in the pending event.""" diff --git a/tests/gateway/test_adapter_startup_secret_scope.py b/tests/gateway/test_adapter_startup_secret_scope.py new file mode 100644 index 000000000000..8e8260436c3d --- /dev/null +++ b/tests/gateway/test_adapter_startup_secret_scope.py @@ -0,0 +1,125 @@ +"""Regression tests — Slack-pattern scoped credential reads at adapter startup. + +Class-closure follow-up to the profile secret-scope cluster (#76462, Slack +pattern #59739, WhatsApp ``_get_wsecret``): the 13 platform adapters below +read credentials at ``__init__`` / availability-check / standalone-send time +with bare ``os.getenv`` (SMS even used ``os.environ[...]``, which KeyErrors). +Under ``gateway.multiplex_profiles`` those reads leak the default profile's +credential into secondary profiles' adapters. + +Each migrated module now carries a module-level ``_get_scoped_secret`` helper +mirroring ``gateway/platforms/whatsapp_common.py::_get_wsecret``: + +- scope installed → scope is authoritative; scoped miss returns the default + (NO borrow from ``os.environ``) +- unscoped under multiplex (the default profile's own startup loop) → + fall back to ``os.getenv`` without raising ``UnscopedSecretError`` + +The helpers are imported and exercised directly rather than constructing every +adapter — construction pulls heavy platform dependencies (slack-bolt-style SDK +imports, webhook servers, sidecars) that the test environment doesn't need. +""" + +import importlib + +import pytest + +from agent import secret_scope as ss + +# (module path, representative credential env var owned by that adapter) +MIGRATED_ADAPTER_MODULES = [ + ("plugins.platforms.irc.adapter", "IRC_SERVER_PASSWORD"), + ("plugins.platforms.line.adapter", "LINE_CHANNEL_ACCESS_TOKEN"), + ("plugins.platforms.teams.adapter", "TEAMS_CLIENT_SECRET"), + ("plugins.platforms.mattermost.adapter", "MATTERMOST_TOKEN"), + ("plugins.platforms.ntfy.adapter", "NTFY_TOKEN"), + ("plugins.platforms.homeassistant.adapter", "HASS_TOKEN"), + ("plugins.platforms.sms.adapter", "TWILIO_AUTH_TOKEN"), + ("plugins.platforms.dingtalk.adapter", "DINGTALK_CLIENT_SECRET"), + ("plugins.platforms.feishu.adapter", "FEISHU_APP_SECRET"), + ("plugins.platforms.wecom.adapter", "WECOM_SECRET"), + ("plugins.platforms.photon.adapter", "PHOTON_PROJECT_SECRET"), + ("plugins.platforms.photon.auth", "PHOTON_PROJECT_SECRET"), + ("plugins.platforms.buzz.adapter", "BUZZ_PRIVATE_KEY"), + ("gateway.platforms.bluebubbles", "BLUEBUBBLES_PASSWORD"), + ("gateway.platforms.api_server", "API_SERVER_KEY"), +] + +MODULE_IDS = [m for m, _ in MIGRATED_ADAPTER_MODULES] + + +@pytest.fixture(autouse=True) +def _reset_multiplex(): + """Multiplex mode off before and after every test.""" + ss.set_multiplex_active(False) + yield + ss.set_multiplex_active(False) + + +def _helper(module_name): + mod = importlib.import_module(module_name) + helper = getattr(mod, "_get_scoped_secret", None) + assert helper is not None, ( + f"{module_name} must define the module-level _get_scoped_secret helper " + "(Slack pattern #59739 / whatsapp_common._get_wsecret)" + ) + return helper + + +@pytest.mark.parametrize(("module_name", "var"), MIGRATED_ADAPTER_MODULES, ids=MODULE_IDS) +def test_helper_exists(module_name, var): + _helper(module_name) + + +@pytest.mark.parametrize(("module_name", "var"), MIGRATED_ADAPTER_MODULES, ids=MODULE_IDS) +def test_scoped_read_wins_over_environ(module_name, var, monkeypatch): + """Scope installed under multiplex: the profile's value wins, not environ's.""" + helper = _helper(module_name) + monkeypatch.setenv(var, "default-profile-value") + ss.set_multiplex_active(True) + tok = ss.set_secret_scope({var: "secondary-profile-value"}) + try: + assert helper(var) == "secondary-profile-value" + finally: + ss.reset_secret_scope(tok) + + +@pytest.mark.parametrize(("module_name", "var"), MIGRATED_ADAPTER_MODULES, ids=MODULE_IDS) +def test_scoped_miss_returns_default_no_environ_borrow(module_name, var, monkeypatch): + """Scope installed but key absent: return the default — never borrow the + default profile's os.environ value into a secondary profile.""" + helper = _helper(module_name) + monkeypatch.setenv(var, "default-profile-value") + ss.set_multiplex_active(True) + tok = ss.set_secret_scope({"SOME_OTHER_KEY": "x"}) + try: + assert helper(var) is None + assert helper(var, "") == "" + assert helper(var, "sentinel") == "sentinel" + finally: + ss.reset_secret_scope(tok) + + +@pytest.mark.parametrize(("module_name", "var"), MIGRATED_ADAPTER_MODULES, ids=MODULE_IDS) +def test_unscoped_under_multiplex_falls_back_to_environ(module_name, var, monkeypatch): + """The DEFAULT profile constructs its adapters unscoped under multiplexing. + A bare get_secret would raise UnscopedSecretError and crash startup; the + helper must fall back to os.environ (that profile's own value) instead.""" + helper = _helper(module_name) + monkeypatch.setenv(var, "default-profile-own-value") + ss.set_multiplex_active(True) + assert ss.current_secret_scope() is None + assert helper(var) == "default-profile-own-value" + + monkeypatch.delenv(var, raising=False) + assert helper(var, "fallback-default") == "fallback-default" + + +@pytest.mark.parametrize(("module_name", "var"), MIGRATED_ADAPTER_MODULES, ids=MODULE_IDS) +def test_single_profile_legacy_environ_read(module_name, var, monkeypatch): + """Multiplex off, no scope: legacy os.environ read keeps working.""" + helper = _helper(module_name) + monkeypatch.setenv(var, "legacy-env-value") + assert helper(var) == "legacy-env-value" + monkeypatch.delenv(var, raising=False) + assert helper(var, "d") == "d" diff --git a/tests/gateway/test_cron_fire_webhook.py b/tests/gateway/test_cron_fire_webhook.py index fe61cea767f6..b8f8a3d4cf17 100644 --- a/tests/gateway/test_cron_fire_webhook.py +++ b/tests/gateway/test_cron_fire_webhook.py @@ -123,3 +123,118 @@ async def test_missing_job_id_400(adapter, monkeypatch): assert spy.fired == [] +@pytest.mark.asyncio +async def test_fire_does_not_require_api_server_key(adapter, monkeypatch): + """The fire endpoint must NOT gate on API_SERVER_KEY — auth is the NAS-JWT. + A request with NO API key header but a valid fire token still succeeds.""" + spy = _SpyProvider() + monkeypatch.setattr("cron.scheduler_provider.resolve_cron_scheduler", lambda: spy) + monkeypatch.setattr( + "plugins.cron_providers.chronos.verify.get_fire_verifier", + lambda: (lambda **kw: {"purpose": "cron_fire"}), + ) + + app = _create_app(adapter) + async with TestClient(TestServer(app)) as cli: + # Bearer is the FIRE token, not the API_SERVER_KEY "sk-secret". + resp = await cli.post("/api/cron/fire", + headers={"Authorization": "Bearer nas-jwt"}, + json={"job_id": "j9"}) + assert resp.status == 202 + for _ in range(50): + if spy.fired: + break + await asyncio.sleep(0.01) + assert spy.fired == ["j9"] + + +@pytest.mark.asyncio +async def test_sync_verifier_runs_off_the_event_loop(adapter, monkeypatch): + """The verifier resolves the signing key from a JWKS URL — a synchronous + HTTP GET on a cache miss. It must run via asyncio.to_thread, NOT inline on + the event loop, or a slow/rate-limited portal stalls every other adapter + sharing the loop. Proof: the sync verifier executes on a worker thread, not + the loop thread. + """ + loop_thread_id = threading.get_ident() + seen = {} + + def blocking_verifier(**kw): + seen["thread_id"] = threading.get_ident() + return {"purpose": "cron_fire"} + + spy = _SpyProvider() + monkeypatch.setattr("cron.scheduler_provider.resolve_cron_scheduler", lambda: spy) + monkeypatch.setattr( + "plugins.cron_providers.chronos.verify.get_fire_verifier", + lambda: blocking_verifier, + ) + + app = _create_app(adapter) + async with TestClient(TestServer(app)) as cli: + resp = await cli.post("/api/cron/fire", + headers={"Authorization": "Bearer good"}, + json={"job_id": "off-loop"}) + assert resp.status == 202 + + # If the verifier had run inline on the loop, its thread id would equal the + # loop thread's; to_thread puts it on a distinct worker thread. + assert seen["thread_id"] != loop_thread_id + + +@pytest.mark.asyncio +async def test_crashing_verifier_fails_closed_401(adapter, monkeypatch): + """A verifier that raises must be treated as a rejection (401), never admit + the fire, and never surface as a 500 — this is the only inbound that can + trigger remote job execution, so it fails closed. + """ + spy = _SpyProvider() + monkeypatch.setattr("cron.scheduler_provider.resolve_cron_scheduler", lambda: spy) + + def exploding_verifier(**kw): + raise RuntimeError("JWKS endpoint unreachable") + + monkeypatch.setattr( + "plugins.cron_providers.chronos.verify.get_fire_verifier", + lambda: exploding_verifier, + ) + + app = _create_app(adapter) + async with TestClient(TestServer(app)) as cli: + resp = await cli.post("/api/cron/fire", + headers={"Authorization": "Bearer boom"}, + json={"job_id": "abc123"}) + assert resp.status == 401 + + await asyncio.sleep(0.05) + assert spy.fired == [] + + +@pytest.mark.asyncio +async def test_async_verifier_is_awaited(adapter, monkeypatch): + """A coroutine verifier (a future async escape-hatch) is awaited directly + rather than dispatched to a thread — a valid async verify still fires. + """ + spy = _SpyProvider() + monkeypatch.setattr("cron.scheduler_provider.resolve_cron_scheduler", lambda: spy) + + async def async_verifier(**kw): + return {"purpose": "cron_fire", "aud": "agent:x"} + + monkeypatch.setattr( + "plugins.cron_providers.chronos.verify.get_fire_verifier", + lambda: async_verifier, + ) + + app = _create_app(adapter) + async with TestClient(TestServer(app)) as cli: + resp = await cli.post("/api/cron/fire", + headers={"Authorization": "Bearer good"}, + json={"job_id": "async-ok"}) + assert resp.status == 202 + + for _ in range(50): + if spy.fired: + break + await asyncio.sleep(0.01) + assert spy.fired == ["async-ok"] diff --git a/tests/gateway/test_email_secret_scope.py b/tests/gateway/test_email_secret_scope.py new file mode 100644 index 000000000000..0dd665efa26b --- /dev/null +++ b/tests/gateway/test_email_secret_scope.py @@ -0,0 +1,248 @@ +"""Tests for email adapter credential isolation under multiplexing. + +Verifies that the email adapter reads EMAIL_ADDRESS, EMAIL_PASSWORD, +EMAIL_IMAP_HOST, and EMAIL_SMTP_HOST from the profile-scoped secret +store (agent.secret_scope.get_secret) instead of os.getenv, so that +a secondary profile in a multiplexed gateway does not inherit the +default profile's email credentials via os.environ. + +Related issues: #50051, #52307 +Related PRs: #51374 (config.py api_server guard), #50094 (config.py scoped env reads) +""" + +import os +import unittest +from unittest.mock import patch, MagicMock + +from agent import secret_scope as ss + + +class TestEmailAdapterSecretScope(unittest.TestCase): + """Verify the email adapter honors the profile secret scope over os.environ.""" + + def setUp(self): + ss.set_multiplex_active(False) + + def tearDown(self): + ss.set_multiplex_active(False) + + @patch.dict(os.environ, { + "EMAIL_ADDRESS": "alpha@test.invalid", + "EMAIL_PASSWORD": "default-pw", + "EMAIL_IMAP_HOST": "imap.default.com", + "EMAIL_SMTP_HOST": "smtp.default.com", + }, clear=False) + def test_adapter_uses_scoped_credentials_not_environ(self): + """When a secret scope is installed, the adapter must read from it, + not from os.environ which may hold another profile's values.""" + from gateway.config import PlatformConfig, Platform + from plugins.platforms.email.adapter import EmailAdapter + + scoped = { + "EMAIL_ADDRESS": "beta@test.invalid", + "EMAIL_PASSWORD": "secondary-pw", + "EMAIL_IMAP_HOST": "imap.secondary.example", + "EMAIL_SMTP_HOST": "smtp.secondary.example", + } + ss.set_multiplex_active(True) + token = ss.set_secret_scope(scoped) + try: + cfg = PlatformConfig(enabled=True) + adapter = EmailAdapter(cfg) + self.assertEqual(adapter._address, "beta@test.invalid") + self.assertEqual(adapter._password, "secondary-pw") + self.assertEqual(adapter._imap_host, "imap.secondary.example") + self.assertEqual(adapter._smtp_host, "smtp.secondary.example") + finally: + ss.reset_secret_scope(token) + + @patch.dict(os.environ, { + "EMAIL_ADDRESS": "alpha@test.invalid", + "EMAIL_PASSWORD": "default-pw", + "EMAIL_IMAP_HOST": "imap.default.com", + "EMAIL_SMTP_HOST": "smtp.default.com", + }, clear=False) + def test_adapter_falls_back_to_environ_without_scope(self): + """Without a secret scope (single-profile mode), the adapter reads + from os.environ — backward-compatible with legacy behavior.""" + from gateway.config import PlatformConfig + from plugins.platforms.email.adapter import EmailAdapter + + cfg = PlatformConfig(enabled=True) + adapter = EmailAdapter(cfg) + self.assertEqual(adapter._address, "alpha@test.invalid") + self.assertEqual(adapter._password, "default-pw") + self.assertEqual(adapter._imap_host, "imap.default.com") + self.assertEqual(adapter._smtp_host, "smtp.default.com") + + @patch.dict(os.environ, { + "EMAIL_ADDRESS": "alpha@test.invalid", + "EMAIL_PASSWORD": "default-pw", + "EMAIL_IMAP_HOST": "imap.default.com", + "EMAIL_SMTP_HOST": "smtp.default.com", + }, clear=False) + def test_check_email_requirements_uses_scope(self): + """check_email_requirements must also honor the secret scope so that + a secondary profile with scoped email creds is detected as configured.""" + scoped = { + "EMAIL_ADDRESS": "beta@test.invalid", + "EMAIL_PASSWORD": "secondary-pw", + "EMAIL_IMAP_HOST": "imap.secondary.example", + "EMAIL_SMTP_HOST": "smtp.secondary.example", + } + ss.set_multiplex_active(True) + token = ss.set_secret_scope(scoped) + try: + from plugins.platforms.email.adapter import check_email_requirements + self.assertTrue(check_email_requirements()) + finally: + ss.reset_secret_scope(token) + + @patch.dict(os.environ, { + "EMAIL_ADDRESS": "alpha@test.invalid", + "EMAIL_PASSWORD": "default-pw", + "EMAIL_IMAP_HOST": "imap.default.com", + "EMAIL_SMTP_HOST": "smtp.default.com", + }, clear=False) + def test_adapter_scoped_missing_key_does_not_leak_environ(self): + """If a key is absent from the scope but present in os.environ, + the adapter must NOT fall through to os.environ (which would leak + the default profile's value).""" + from gateway.config import PlatformConfig + from plugins.platforms.email.adapter import EmailAdapter + + scoped = { + "EMAIL_ADDRESS": "beta@test.invalid", + # EMAIL_PASSWORD intentionally missing from scope + "EMAIL_IMAP_HOST": "imap.secondary.example", + "EMAIL_SMTP_HOST": "smtp.secondary.example", + } + ss.set_multiplex_active(True) + token = ss.set_secret_scope(scoped) + try: + cfg = PlatformConfig(enabled=True) + adapter = EmailAdapter(cfg) + self.assertEqual(adapter._address, "beta@test.invalid") + # Password must NOT be the default profile's "default-pw" + self.assertNotEqual(adapter._password, "default-pw") + self.assertEqual(adapter._password, "") + finally: + ss.reset_secret_scope(token) + + @patch.dict(os.environ, { + "EMAIL_ADDRESS": "alpha@test.invalid", + "EMAIL_PASSWORD": "default-pw", + "EMAIL_IMAP_HOST": "imap.default.com", + "EMAIL_SMTP_HOST": "smtp.default.com", + "EMAIL_ALLOWED_USERS": "gamma@test.invalid", + }, clear=False) + def test_allowed_users_uses_scope(self): + """EMAIL_ALLOWED_USERS must also be read from the secret scope + so a secondary profile gets its own allowlist, not the default's.""" + scoped = { + "EMAIL_ADDRESS": "beta@test.invalid", + "EMAIL_PASSWORD": "secondary-pw", + "EMAIL_IMAP_HOST": "imap.secondary.example", + "EMAIL_SMTP_HOST": "smtp.secondary.example", + "EMAIL_ALLOWED_USERS": "epsilon@test.invalid,delta@test.invalid", + } + ss.set_multiplex_active(True) + token = ss.set_secret_scope(scoped) + try: + # _allowlist_in_effect reads EMAIL_ALLOWED_USERS — verify it + # sees the scoped value, not the environ value + from plugins.platforms.email.adapter import EmailAdapter + self.assertTrue(EmailAdapter._allowlist_in_effect()) + finally: + ss.reset_secret_scope(token) + + +class TestEmailAdapterUnscopedUnderMultiplex(unittest.TestCase): + """The DEFAULT profile's adapter constructs UNSCOPED under multiplexing. + + In a multiplexed gateway only secondary profiles run inside + ``_profile_runtime_scope``; the default profile's adapter is constructed + with no scope installed while multiplexing is active. A bare + ``get_secret`` raises ``UnscopedSecretError`` there and would crash the + email path on startup — the exact WhatsApp defect fixed in 5438e9c629. + The Slack-pattern helper (``_get_esecret``) must swallow that and read + the default profile's own ``os.environ`` values instead. + """ + + def setUp(self): + ss.set_multiplex_active(False) + + def tearDown(self): + ss.set_multiplex_active(False) + + @patch.dict(os.environ, { + "EMAIL_ADDRESS": "alpha@test.invalid", + "EMAIL_PASSWORD": "default-pw", + "EMAIL_IMAP_HOST": "imap.default.com", + "EMAIL_SMTP_HOST": "smtp.default.com", + "EMAIL_IMAP_PORT": "1993", + "EMAIL_SMTP_PORT": "1587", + "EMAIL_POLL_INTERVAL": "30", + }, clear=False) + def test_default_profile_constructs_unscoped_under_multiplex(self): + """Multiplex ON + no scope: construction must not raise and must + read the default profile's own environ values.""" + from gateway.config import PlatformConfig + from plugins.platforms.email.adapter import EmailAdapter + + ss.set_multiplex_active(True) + cfg = PlatformConfig(enabled=True) + adapter = EmailAdapter(cfg) # must NOT raise UnscopedSecretError + self.assertEqual(adapter._address, "alpha@test.invalid") + self.assertEqual(adapter._password, "default-pw") + self.assertEqual(adapter._imap_host, "imap.default.com") + self.assertEqual(adapter._smtp_host, "smtp.default.com") + self.assertEqual(adapter._imap_port, 1993) + self.assertEqual(adapter._smtp_port, 1587) + self.assertEqual(adapter._poll_interval, 30) + + @patch.dict(os.environ, { + "EMAIL_ADDRESS": "alpha@test.invalid", + "EMAIL_PASSWORD": "default-pw", + "EMAIL_IMAP_HOST": "imap.default.com", + "EMAIL_SMTP_HOST": "smtp.default.com", + "EMAIL_IMAP_PORT": "1993", + "EMAIL_SMTP_PORT": "1587", + "EMAIL_POLL_INTERVAL": "30", + "EMAIL_TRUST_FROM_HEADER": "true", + }, clear=False) + def test_scoped_ports_and_trust_flag_do_not_inherit_environ(self): + """The env_int variants (EMAIL_IMAP_PORT / EMAIL_SMTP_PORT / + EMAIL_POLL_INTERVAL) and EMAIL_TRUST_FROM_HEADER must resolve from + the installed scope, not from the primary profile's environ.""" + from gateway.config import PlatformConfig + from plugins.platforms.email.adapter import EmailAdapter + + scoped = { + "EMAIL_ADDRESS": "beta@test.invalid", + "EMAIL_PASSWORD": "secondary-pw", + "EMAIL_IMAP_HOST": "imap.secondary.example", + "EMAIL_SMTP_HOST": "smtp.secondary.example", + "EMAIL_IMAP_PORT": "2993", + "EMAIL_SMTP_PORT": "2587", + "EMAIL_POLL_INTERVAL": "60", + # scope does NOT opt into trusting From: — environ's "true" + # must not leak in. + } + ss.set_multiplex_active(True) + token = ss.set_secret_scope(scoped) + try: + cfg = PlatformConfig(enabled=True) + adapter = EmailAdapter(cfg) + self.assertEqual(adapter._imap_port, 2993) + self.assertEqual(adapter._smtp_port, 2587) + self.assertEqual(adapter._poll_interval, 60) + # Environ's EMAIL_TRUST_FROM_HEADER=true must not leak in: the + # scope did not opt out, so authentication stays required. + self.assertTrue(adapter._require_authenticated_sender) + finally: + ss.reset_secret_scope(token) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/gateway/test_memory_trim_housekeeping.py b/tests/gateway/test_memory_trim_housekeeping.py new file mode 100644 index 000000000000..c8818f36b369 --- /dev/null +++ b/tests/gateway/test_memory_trim_housekeeping.py @@ -0,0 +1,32 @@ +"""Memory-trim coverage for the long-lived messaging gateway housekeeper.""" + +import gateway.run as gateway_run + + +class _OneTickStopEvent: + """Run one housekeeping tick without a sleep or background thread.""" + + def __init__(self): + self.waited = False + + def is_set(self): + return self.waited + + def wait(self, timeout=None): + self.waited = True + return True + + +def test_gateway_housekeeping_calls_periodic_memory_trim(monkeypatch): + import hermes_cli.mem_trim as mem_trim + + calls = [] + monkeypatch.setattr( + mem_trim, + "trim_memory", + lambda **kwargs: calls.append(kwargs) or True, + ) + + gateway_run._start_gateway_housekeeping(_OneTickStopEvent(), interval=0) + + assert calls == [{"reason": "messaging gateway housekeeping"}] diff --git a/tests/gateway/test_qqbot_credential_isolation.py b/tests/gateway/test_qqbot_credential_isolation.py new file mode 100644 index 000000000000..5723ba03f6da --- /dev/null +++ b/tests/gateway/test_qqbot_credential_isolation.py @@ -0,0 +1,125 @@ +"""Credential isolation for the QQ (qqbot) gateway adapter. + +Covers the multiplex credential-collision class (same class as the +WeChat/weixin adapter tracked in #59662): the QQ adapter resolves its ``QQ_*`` +settings through the active profile secret scope rather than raw ``os.getenv``, +so a secondary profile whose secret lives in its own ``.env`` (installed as an +isolated scope, not into ``os.environ``) does not fall back to the +default/primary profile's value. + +Also guards the primary/active profile: it is constructed without a scope and +legitimately owns ``os.environ``, so the resolver must fall back to +``os.environ`` there (not fail closed) even when multiplexing is active — +otherwise the active profile's adapter would raise ``UnscopedSecretError`` on +construction and fail to start. +""" +import pytest + +from agent import secret_scope as ss +from gateway.config import PlatformConfig +from gateway.platforms.qqbot.adapter import QQAdapter + + +@pytest.fixture(autouse=True) +def _reset_multiplex(): + ss.set_multiplex_active(False) + yield + ss.set_multiplex_active(False) + + +def _make_adapter(extra=None): + return QQAdapter(PlatformConfig(enabled=True, extra=extra or {})) + + +class TestQQCredentialScope: + def test_credentials_read_scope_not_environ(self, monkeypatch): + # os.environ holds another profile's values; the scoped values must win. + monkeypatch.setenv("QQ_APP_ID", "global-app") + monkeypatch.setenv("QQ_CLIENT_SECRET", "global-secret") + ss.set_multiplex_active(True) + tok = ss.set_secret_scope( + {"QQ_APP_ID": "profileA-app", "QQ_CLIENT_SECRET": "profileA-secret"} + ) + try: + adapter = _make_adapter() + finally: + ss.reset_secret_scope(tok) + assert adapter._app_id == "profileA-app" + assert adapter._client_secret == "profileA-secret" + + def test_two_profiles_isolated(self): + ss.set_multiplex_active(True) + tok_a = ss.set_secret_scope({"QQ_CLIENT_SECRET": "secret-A"}) + try: + a = _make_adapter() + finally: + ss.reset_secret_scope(tok_a) + tok_b = ss.set_secret_scope({"QQ_CLIENT_SECRET": "secret-B"}) + try: + b = _make_adapter() + finally: + ss.reset_secret_scope(tok_b) + assert a._client_secret == "secret-A" + assert b._client_secret == "secret-B" + + def test_single_profile_still_reads_environ(self, monkeypatch): + # No scope + multiplex inactive (default single-profile deployment): + # legacy os.environ behavior is preserved — no regression. + monkeypatch.setenv("QQ_CLIENT_SECRET", "legacy-secret") + adapter = _make_adapter() + assert adapter._client_secret == "legacy-secret" + + def test_active_profile_no_scope_reads_environ_without_raising(self, monkeypatch): + # The primary/active profile is built with NO scope while multiplexing + # is active. A bare get_secret() would fail closed (UnscopedSecretError) + # and break its startup; the resolver must fall back to os.environ. + monkeypatch.setenv("QQ_APP_ID", "primary-app") + monkeypatch.setenv("QQ_CLIENT_SECRET", "primary-secret") + ss.set_multiplex_active(True) + assert ss.current_secret_scope() is None # no scope installed + adapter = _make_adapter() # must not raise + assert adapter._app_id == "primary-app" + assert adapter._client_secret == "primary-secret" + + def test_explicit_config_extra_takes_precedence(self, monkeypatch): + # An explicit value in config.extra still wins over env/scope. + monkeypatch.setenv("QQ_CLIENT_SECRET", "env-secret") + adapter = _make_adapter(extra={"client_secret": "explicit"}) + assert adapter._client_secret == "explicit" + + +class TestQQSttConfigScope: + def test_stt_api_key_reads_scope(self, monkeypatch): + monkeypatch.setenv("QQ_STT_API_KEY", "global-stt-key") + ss.set_multiplex_active(True) + tok = ss.set_secret_scope({"QQ_STT_API_KEY": "profileA-stt-key"}) + try: + adapter = _make_adapter() + stt = adapter._resolve_stt_config() + finally: + ss.reset_secret_scope(tok) + assert stt is not None + assert stt["api_key"] == "profileA-stt-key" + + def test_all_stt_values_read_scope(self, monkeypatch): + # Every QQ_STT_* value must come from the scope, not os.environ. + monkeypatch.setenv("QQ_STT_API_KEY", "global-stt-key") + monkeypatch.setenv("QQ_STT_BASE_URL", "https://global.example/v1") + monkeypatch.setenv("QQ_STT_MODEL", "global-asr") + ss.set_multiplex_active(True) + tok = ss.set_secret_scope( + { + "QQ_STT_API_KEY": "profileA-stt-key", + "QQ_STT_BASE_URL": "https://scoped.example/v1", + "QQ_STT_MODEL": "scoped-asr", + } + ) + try: + adapter = _make_adapter() + stt = adapter._resolve_stt_config() + finally: + ss.reset_secret_scope(tok) + assert stt is not None + assert stt["api_key"] == "profileA-stt-key" + assert stt["base_url"] == "https://scoped.example/v1" + assert stt["model"] == "scoped-asr" diff --git a/tests/gateway/test_qqbot_scope_paths.py b/tests/gateway/test_qqbot_scope_paths.py new file mode 100644 index 000000000000..11a2ab70c050 --- /dev/null +++ b/tests/gateway/test_qqbot_scope_paths.py @@ -0,0 +1,268 @@ +"""End-to-end profile-scope coverage for the QQ (qqbot) authorization, +startup-validation, and direct-send paths. + +Complements ``test_qqbot_credential_isolation.py`` (adapter-level resolver): +the adapter intake fix alone is not enough because three other paths read the +same per-profile ``QQ_*`` values independently: + +- gateway authorization (``AuthorizationMixin._is_user_authorized``) reads + ``QQ_ALLOW_ALL_USERS`` when deciding whether to honor an allow-all opt-in; +- secondary-profile startup validation + (``gateway.run._own_policy_open_startup_violation``) reads the platform + opt-in while running inside ``_profile_runtime_scope``; +- the ``send_message`` tool's direct REST path (``_send_qqbot``) falls back to + ``QQ_APP_ID`` / ``QQ_CLIENT_SECRET``. + +Each must resolve through the active profile secret scope (scope wins over +``os.environ``; a profile that did NOT opt in must not inherit the primary +profile's environ opt-in) while unscoped single-profile deployments keep the +legacy ``os.environ`` behavior. +""" + +import sys +import types +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from agent import secret_scope as ss +from gateway.config import GatewayConfig, Platform, PlatformConfig +from gateway.session import SessionSource + + +@pytest.fixture(autouse=True) +def _reset_scope_state(monkeypatch): + for key in ( + "QQ_ALLOW_ALL_USERS", + "QQ_ALLOWED_USERS", + "QQ_GROUP_ALLOWED_USERS", + "QQ_APP_ID", + "QQ_CLIENT_SECRET", + "GATEWAY_ALLOWED_USERS", + "GATEWAY_ALLOW_ALL_USERS", + ): + monkeypatch.delenv(key, raising=False) + ss.set_multiplex_active(False) + yield + ss.set_multiplex_active(False) + + +def _make_qq_runner(): + """Minimal runner whose authz path reaches the QQ env checks.""" + from gateway.run import GatewayRunner + + runner = object.__new__(GatewayRunner) + runner.config = GatewayConfig(multiplex_profiles=True) + + default_adapter = SimpleNamespace( + send=AsyncMock(), + enforces_own_access_policy=True, + _dm_policy="allowlist", + _group_policy="pairing", + ) + secondary_adapter = SimpleNamespace( + send=AsyncMock(), + enforces_own_access_policy=True, + _dm_policy="open", + _group_policy="open", + ) + runner.adapters = {Platform.QQBOT: default_adapter} + runner._profile_adapters = {"coder": {Platform.QQBOT: secondary_adapter}} + runner.pairing_store = MagicMock() + runner.pairing_store.is_approved.return_value = False + return runner + + +def _qq_dm_source(profile="coder"): + return SessionSource( + platform=Platform.QQBOT, + user_id="user-1", + chat_id="dm-chat", + user_name="user-1", + chat_type="dm", + profile=profile, + ) + + +class TestAuthzAllowAllScope: + def test_scoped_allow_all_honored(self): + # The secondary profile opted in via its own .env (scope); os.environ + # has no opt-in. Authorization must honor the scoped value. + runner = _make_qq_runner() + ss.set_multiplex_active(True) + tok = ss.set_secret_scope({"QQ_ALLOW_ALL_USERS": "true"}) + try: + assert runner._is_user_authorized(_qq_dm_source()) is True + finally: + ss.reset_secret_scope(tok) + + @pytest.mark.xfail( + reason=( + "gateway/authz_mixin.py still reads the platform allow-all flag via " + "_auth_env, which falls through to os.environ on a scoped miss; the " + "scope-authoritative gate (_platform_gate_env semantics) for the " + "remaining authz_mixin reads lands in a separate PR. Flips green " + "when that PR converts the allow-all read." + ), + strict=True, + ) + def test_scope_does_not_inherit_environ_opt_in(self, monkeypatch): + # The PRIMARY profile opted in via os.environ; the secondary profile's + # scope has no opt-in. The secondary must NOT inherit the primary's + # allow-all (this is the cross-profile leak the fix closes). + monkeypatch.setenv("QQ_ALLOW_ALL_USERS", "true") + runner = _make_qq_runner() + ss.set_multiplex_active(True) + tok = ss.set_secret_scope({}) + try: + assert runner._is_user_authorized(_qq_dm_source()) is False + finally: + ss.reset_secret_scope(tok) + + def test_single_profile_environ_unchanged(self, monkeypatch): + # Multiplex inactive, no scope: legacy os.environ behavior preserved. + monkeypatch.setenv("QQ_ALLOW_ALL_USERS", "true") + runner = _make_qq_runner() + assert runner._is_user_authorized(_qq_dm_source(profile=None)) is True + + +class TestAuthzAllowlistScope: + """Pins the per-platform user-allowlist read (not just the allow-all flag). + + Distinct from the allow-all tests: here QQ_ALLOW_ALL_USERS is absent, so + authorization depends entirely on the scoped QQ_ALLOWED_USERS matching the + sender — reverting that read to raw os.getenv would default-deny. + """ + + def test_scoped_user_allowlist_authorizes(self, monkeypatch): + monkeypatch.setenv("QQ_ALLOWED_USERS", "") # global env has no allowlist + runner = _make_qq_runner() + ss.set_multiplex_active(True) + tok = ss.set_secret_scope({"QQ_ALLOWED_USERS": "user-1,user-2"}) + try: + assert runner._is_user_authorized(_qq_dm_source()) is True + finally: + ss.reset_secret_scope(tok) + + def test_scoped_user_allowlist_excludes_non_member(self, monkeypatch): + monkeypatch.setenv("QQ_ALLOWED_USERS", "user-1") # primary env admits user-1 + runner = _make_qq_runner() + ss.set_multiplex_active(True) + # Secondary scope lists only user-9 → user-1 must NOT inherit the + # primary's environ allowlist. + tok = ss.set_secret_scope({"QQ_ALLOWED_USERS": "user-9"}) + try: + assert runner._is_user_authorized(_qq_dm_source()) is False + finally: + ss.reset_secret_scope(tok) + + +class TestStartupValidatorScope: + @staticmethod + def _open_dm_config(): + return GatewayConfig( + platforms={ + Platform.QQBOT: PlatformConfig( + enabled=True, extra={"dm_policy": "open"} + ) + } + ) + + def test_scoped_opt_in_clears_violation(self): + # Mirrors _start_one_profile_adapters: the validator runs inside the + # profile scope, so the profile's own opt-in must clear the violation. + from gateway.run import _own_policy_open_startup_violation + + ss.set_multiplex_active(True) + tok = ss.set_secret_scope({"QQ_ALLOW_ALL_USERS": "true"}) + try: + assert _own_policy_open_startup_violation(self._open_dm_config()) is None + finally: + ss.reset_secret_scope(tok) + + def test_scope_does_not_inherit_environ_opt_in(self, monkeypatch): + # Primary's environ opt-in must not silently bless a secondary + # profile whose own scope never opted in. + from gateway.run import _own_policy_open_startup_violation + + monkeypatch.setenv("QQ_ALLOW_ALL_USERS", "true") + ss.set_multiplex_active(True) + tok = ss.set_secret_scope({}) + try: + violation = _own_policy_open_startup_violation(self._open_dm_config()) + finally: + ss.reset_secret_scope(tok) + assert violation is not None + assert "qqbot" in violation + + def test_unscoped_environ_unchanged(self, monkeypatch): + # Single-profile startup (no scope installed) keeps reading environ. + from gateway.run import _own_policy_open_startup_violation + + monkeypatch.setenv("QQ_ALLOW_ALL_USERS", "true") + assert _own_policy_open_startup_violation(self._open_dm_config()) is None + + +class TestDirectSendScope: + @staticmethod + def _fake_httpx(captured): + class _Resp: + status_code = 500 + + @staticmethod + def json(): + return {} + + class _AsyncClient: + def __init__(self, *args, **kwargs): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + async def post(self, url, **kwargs): + captured.append(kwargs.get("json") or {}) + return _Resp() + + module = types.ModuleType("httpx") + module.AsyncClient = _AsyncClient + return module + + @pytest.mark.asyncio + async def test_scoped_credentials_win_over_environ(self, monkeypatch): + from tools.send_message_tool import _send_qqbot + + captured = [] + monkeypatch.setitem(sys.modules, "httpx", self._fake_httpx(captured)) + monkeypatch.setenv("QQ_APP_ID", "global-app") + monkeypatch.setenv("QQ_CLIENT_SECRET", "global-secret") + ss.set_multiplex_active(True) + tok = ss.set_secret_scope( + {"QQ_APP_ID": "profileA-app", "QQ_CLIENT_SECRET": "profileA-secret"} + ) + try: + await _send_qqbot( + PlatformConfig(enabled=True, extra={}), "chat-1", "hi" + ) + finally: + ss.reset_secret_scope(tok) + assert captured, "token request never issued" + assert captured[0]["appId"] == "profileA-app" + assert captured[0]["clientSecret"] == "profileA-secret" + + @pytest.mark.asyncio + async def test_unscoped_falls_back_to_environ(self, monkeypatch): + from tools.send_message_tool import _send_qqbot + + captured = [] + monkeypatch.setitem(sys.modules, "httpx", self._fake_httpx(captured)) + monkeypatch.setenv("QQ_APP_ID", "env-app") + monkeypatch.setenv("QQ_CLIENT_SECRET", "env-secret") + await _send_qqbot(PlatformConfig(enabled=True, extra={}), "chat-1", "hi") + assert captured, "token request never issued" + assert captured[0]["appId"] == "env-app" + assert captured[0]["clientSecret"] == "env-secret" diff --git a/tests/gateway/test_routing_save_fast_path.py b/tests/gateway/test_routing_save_fast_path.py new file mode 100644 index 000000000000..d693c3b236d4 --- /dev/null +++ b/tests/gateway/test_routing_save_fast_path.py @@ -0,0 +1,445 @@ +"""Single-row routing save fast path. + +Metadata-only per-turn writes (get_or_create_session's healthy-path +``updated_at`` bump, ``update_session``) persist through a single-row +UPSERT instead of rewriting the whole routing index. These tests prove +the write-skipping never loses a NEEDED write: changed values always +land in state.db, and restart rebinding works even when the legacy +sessions.json mirror lagged behind (or never existed). +""" +from __future__ import annotations + +import json +import threading + +import hermes_state +from gateway.config import GatewayConfig, Platform +from gateway.session import SessionSource, SessionStore + + +def _source(user_id: str = "user-1") -> SessionSource: + return SessionSource( + platform=Platform.LOCAL, + chat_id="cli", + chat_name="CLI", + chat_type="dm", + user_id=user_id, + ) + + +def _make_store(tmp_path, monkeypatch, **config_kwargs) -> SessionStore: + monkeypatch.setattr(hermes_state, "DEFAULT_DB_PATH", tmp_path / "state.db") + return SessionStore( + sessions_dir=tmp_path / "sessions", + config=GatewayConfig(**config_kwargs), + ) + + +def _routing_row(store: SessionStore, session_key: str) -> dict: + rows = store._db.load_gateway_routing_entries(scope=store._routing_scope()) + return json.loads(rows[session_key]) + + +class TestChangedValuesAlwaysPersist: + def test_update_session_persists_last_prompt_tokens_to_db( + self, tmp_path, monkeypatch + ): + store = _make_store(tmp_path, monkeypatch) + entry = store.get_or_create_session(_source()) + + store.update_session(entry.session_key, last_prompt_tokens=54321) + + durable = _routing_row(store, entry.session_key) + assert durable["last_prompt_tokens"] == 54321 + store._db.close() + + def test_healthy_path_bump_persists_updated_at_to_db( + self, tmp_path, monkeypatch + ): + store = _make_store(tmp_path, monkeypatch) + entry = store.get_or_create_session(_source()) + before = _routing_row(store, entry.session_key)["updated_at"] + + # Second lookup takes the healthy-path metadata-only save. + again = store.get_or_create_session(_source()) + assert again.session_id == entry.session_id + + after = _routing_row(store, entry.session_key)["updated_at"] + assert after >= before + # The row reflects the in-memory bump, not a stale copy. + assert after == again.updated_at.isoformat() + store._db.close() + + def test_fast_path_survives_restart_across_stores( + self, tmp_path, monkeypatch + ): + store = _make_store(tmp_path, monkeypatch) + entry = store.get_or_create_session(_source()) + store.update_session(entry.session_key, last_prompt_tokens=777) + store._db.close() + + restarted = _make_store(tmp_path, monkeypatch) + rebound = restarted.get_or_create_session(_source()) + assert rebound.session_id == entry.session_id + assert rebound.last_prompt_tokens == 777 + restarted._db.close() + + +class TestRestartRebindWithoutMirror: + def test_rebind_works_when_mirror_lagged_fast_path_writes( + self, tmp_path, monkeypatch + ): + """The fast path skips sessions.json; state.db alone must rebind.""" + store = _make_store(tmp_path, monkeypatch) + entry = store.get_or_create_session(_source()) + sessions_json = tmp_path / "sessions" / "sessions.json" + assert sessions_json.exists() # structural save wrote the mirror + + # Remove the mirror, then do fast-path-only writes: they must NOT + # recreate it (proves the fast path ran) and must not need it. + sessions_json.unlink() + store.update_session(entry.session_key, last_prompt_tokens=42) + again = store.get_or_create_session(_source()) + assert again.session_id == entry.session_id + assert not sessions_json.exists() + store._db.close() + + restarted = _make_store(tmp_path, monkeypatch) + rebound = restarted.get_or_create_session(_source()) + assert rebound.session_id == entry.session_id + assert rebound.last_prompt_tokens == 42 + restarted._db.close() + + def test_compression_heal_takes_full_path_and_updates_mirror( + self, tmp_path, monkeypatch + ): + """A heal rewrites session_id, so it must bypass the fast path. + + The fast path persists state.db only; if a heal took it, the + sessions.json mirror would keep the ended parent id until the + next structural save (indefinitely on a healthy key). + """ + store = _make_store(tmp_path, monkeypatch) + entry = store.get_or_create_session(_source()) + healed_id = entry.session_id + "_child" + monkeypatch.setattr( + store, + "_compression_tip_for_session_id", + lambda sid: healed_id if sid == entry.session_id else sid, + ) + + again = store.get_or_create_session(_source()) + assert again.session_id == healed_id + + assert _routing_row(store, entry.session_key)["session_id"] == healed_id + sessions_json = tmp_path / "sessions" / "sessions.json" + data = json.loads(sessions_json.read_text(encoding="utf-8")) + assert data[entry.session_key]["session_id"] == healed_id + store._db.close() + + def test_structural_save_still_rewrites_mirror(self, tmp_path, monkeypatch): + store = _make_store(tmp_path, monkeypatch) + first = store.get_or_create_session(_source()) + fresh = store.get_or_create_session(_source(), force_new=True) + assert fresh.session_id != first.session_id + + sessions_json = tmp_path / "sessions" / "sessions.json" + data = json.loads(sessions_json.read_text(encoding="utf-8")) + assert data[fresh.session_key]["session_id"] == fresh.session_id + store._db.close() + + +class TestFallbacks: + def test_no_db_falls_back_to_full_rewrite(self, tmp_path, monkeypatch): + """DB-less installs keep sessions.json durable every turn.""" + store = _make_store(tmp_path, monkeypatch) + entry = store.get_or_create_session(_source()) + store._db.close() + store._db = None + + store.update_session(entry.session_key, last_prompt_tokens=99) + + sessions_json = tmp_path / "sessions" / "sessions.json" + data = json.loads(sessions_json.read_text(encoding="utf-8")) + assert data[entry.session_key]["last_prompt_tokens"] == 99 + + def test_failed_upsert_falls_back_to_full_rewrite( + self, tmp_path, monkeypatch + ): + store = _make_store(tmp_path, monkeypatch) + entry = store.get_or_create_session(_source()) + + def boom(*args, **kwargs): + raise RuntimeError("disk on fire") + + monkeypatch.setattr(store._db, "save_gateway_routing_entry", boom) + store.update_session(entry.session_key, last_prompt_tokens=1234) + + # The full rewrite carried the change to both stores. + sessions_json = tmp_path / "sessions" / "sessions.json" + data = json.loads(sessions_json.read_text(encoding="utf-8")) + assert data[entry.session_key]["last_prompt_tokens"] == 1234 + assert _routing_row(store, entry.session_key)["last_prompt_tokens"] == 1234 + store._db.close() + + +class TestPeerRecordConsistency: + def test_update_session_records_peer_fields_snapshotted_under_lock( + self, tmp_path, monkeypatch + ): + """Peer fields must come from one lock-held snapshot, not late reads. + + The peer record runs outside ``_lock``; a concurrent reset that + rewrites the entry in that window must not produce a torn + old/new field mix in the recorded row. + """ + store = _make_store(tmp_path, monkeypatch) + entry = store.get_or_create_session(_source()) + original_id = entry.session_id + + recorded = [] + monkeypatch.setattr( + store, + "_record_gateway_session_peer", + lambda sid, key, origin, display_name=None: recorded.append( + (sid, key, display_name) + ), + ) + orig_save_entry = store._save_entry + + def swap_then_save(key): + # Simulate a concurrent reset landing after update_session + # released _lock but before the peer record runs. + with store._lock: + store._entries[key].session_id = "reset-rewrote-me" + orig_save_entry(key) + + monkeypatch.setattr(store, "_save_entry", swap_then_save) + store.update_session(entry.session_key, last_prompt_tokens=5) + + assert recorded == [ + (original_id, entry.session_key, entry.display_name) + ] + store._db.close() + + +class TestGenerationOrdering: + def test_upsert_skipped_when_newer_full_snapshot_persisted( + self, tmp_path, monkeypatch + ): + """A full snapshot taken after our serialize point must win. + + Its copy of the key is same-or-newer, so writing ours would + regress it. Simulated by advancing _persisted_routing_generation + past the generation captured at serialize time. + """ + store = _make_store(tmp_path, monkeypatch) + entry = store.get_or_create_session(_source()) + + calls = [] + monkeypatch.setattr( + store._db, + "save_gateway_routing_entry", + lambda *a, **k: calls.append((a, k)), + ) + store._persisted_routing_generation = store._routing_generation + 1 + + store._save_entry(entry.session_key) + + assert calls == [] + store._db.close() + + def test_restart_rebind_after_skipped_idempotent_write( + self, tmp_path, monkeypatch + ): + """A skipped fast-path write never orphans the session on restart. + + The skip only fires when a newer FULL snapshot — which contains + this key — already persisted, so state.db can always rebind. + """ + store = _make_store(tmp_path, monkeypatch) + entry = store.get_or_create_session(_source()) + + calls = [] + monkeypatch.setattr( + store._db, + "save_gateway_routing_entry", + lambda *a, **k: calls.append(a), + ) + store._persisted_routing_generation = store._routing_generation + 1 + store._save_entry(entry.session_key) + assert calls == [] # the idempotent write was skipped + store._db.close() + + restarted = _make_store(tmp_path, monkeypatch) + rebound = restarted.get_or_create_session(_source()) + assert rebound.session_id == entry.session_id + restarted._db.close() + + def test_upsert_proceeds_at_current_generation(self, tmp_path, monkeypatch): + store = _make_store(tmp_path, monkeypatch) + entry = store.get_or_create_session(_source()) + + calls = [] + monkeypatch.setattr( + store._db, + "save_gateway_routing_entry", + lambda key, entry_json, **k: calls.append((key, entry_json, k)), + ) + + store._save_entry(entry.session_key) + + assert len(calls) == 1 + key, entry_json, kwargs = calls[0] + assert key == entry.session_key + assert json.loads(entry_json)["session_id"] == entry.session_id + assert kwargs["scope"] == store._routing_scope() + store._db.close() + + def test_missing_key_is_a_noop(self, tmp_path, monkeypatch): + store = _make_store(tmp_path, monkeypatch) + store._ensure_loaded() + + calls = [] + monkeypatch.setattr( + store._db, + "save_gateway_routing_entry", + lambda *a, **k: calls.append(a), + ) + + store._save_entry("agent:main:local:nope") + + assert calls == [] + store._db.close() + + +class _GatedSaveLock: + """``_save_lock`` wrapper that parks one thread at lock entry. + + The parked thread has already serialized its entry (and taken its + revision) under ``_lock``, so this deterministically reproduces a + delayed write: the writer sits between its serialize point and the + durable-write section while other writers run to completion. + """ + + def __init__(self, inner: threading.Lock) -> None: + self._inner = inner + self.gated_thread: threading.Thread | None = None + self.reached = threading.Event() + self.release = threading.Event() + + def __enter__(self): + if threading.current_thread() is self.gated_thread: + self.reached.set() + assert self.release.wait(timeout=5), "gated writer never released" + return self._inner.__enter__() + + def __exit__(self, *exc): + return self._inner.__exit__(*exc) + + +class TestDelayedWriteOrdering: + def test_reverse_completion_fast_saves_keep_newer_entry( + self, tmp_path, monkeypatch + ): + """Two same-key fast saves completing in reverse order. + + The older save serializes first but its UPSERT is delayed past + the newer save's. Its revision is below the newer one recorded + in ``_fast_persisted_entries``, so it must skip — the newer + entry_json stays in state.db. + """ + store = _make_store(tmp_path, monkeypatch) + entry = store.get_or_create_session(_source()) + + gate = _GatedSaveLock(store._save_lock) + store._save_lock = gate + older = threading.Thread( + target=store.update_session, + args=(entry.session_key,), + kwargs={"last_prompt_tokens": 1}, + ) + gate.gated_thread = older + older.start() + # The older save has serialized last_prompt_tokens=1 and parked + # before its UPSERT; the newer save now runs to completion. + assert gate.reached.wait(timeout=5) + store.update_session(entry.session_key, last_prompt_tokens=2) + + gate.release.set() + older.join(timeout=5) + assert not older.is_alive() + + assert _routing_row(store, entry.session_key)["last_prompt_tokens"] == 2 + store._db.close() + + def test_delayed_full_rewrite_folds_in_newer_fast_save( + self, tmp_path, monkeypatch + ): + """A full rewrite landing after a later-serialized fast save. + + The rewrite's snapshot predates the fast save, so replaying it + verbatim would regress the key. ``_persist_routing_data`` must + fold the newer fast record into the rewrite — in state.db and in + the sessions.json mirror. + """ + store = _make_store(tmp_path, monkeypatch) + entry = store.get_or_create_session(_source()) + + with store._lock: + data, generation = store._snapshot_routing_locked() + store.update_session(entry.session_key, last_prompt_tokens=7) + + # The delayed full rewrite lands last. + store._persist_routing_data(data, generation) + + assert _routing_row(store, entry.session_key)["last_prompt_tokens"] == 7 + sessions_json = tmp_path / "sessions" / "sessions.json" + mirror = json.loads(sessions_json.read_text(encoding="utf-8")) + assert mirror[entry.session_key]["last_prompt_tokens"] == 7 + store._db.close() + + def test_delayed_fast_save_skips_after_newer_full_rewrite( + self, tmp_path, monkeypatch + ): + """A fast save delayed past a full rewrite serialized after it. + + The rewrite's snapshot contains a newer copy of the key, so the + parked UPSERT must skip instead of regressing it. + """ + store = _make_store(tmp_path, monkeypatch) + entry = store.get_or_create_session(_source()) + + upserts = [] + real_saver = store._db.save_gateway_routing_entry + + def counting_saver(session_key, entry_json, **kwargs): + upserts.append(session_key) + real_saver(session_key, entry_json, **kwargs) + + monkeypatch.setattr( + store._db, "save_gateway_routing_entry", counting_saver + ) + gate = _GatedSaveLock(store._save_lock) + store._save_lock = gate + older = threading.Thread( + target=store.update_session, + args=(entry.session_key,), + kwargs={"last_prompt_tokens": 1}, + ) + gate.gated_thread = older + older.start() + assert gate.reached.wait(timeout=5) + + # A full rewrite serialized after the parked save persists first. + with store._lock: + store._entries[entry.session_key].last_prompt_tokens = 2 + store._save_entries() + + gate.release.set() + older.join(timeout=5) + assert not older.is_alive() + + assert upserts == [] # the delayed UPSERT was skipped + assert _routing_row(store, entry.session_key)["last_prompt_tokens"] == 2 + store._db.close() diff --git a/tests/gateway/test_voice_command.py b/tests/gateway/test_voice_command.py index b2d4ab6832a8..4a988a6e036a 100644 --- a/tests/gateway/test_voice_command.py +++ b/tests/gateway/test_voice_command.py @@ -1947,3 +1947,54 @@ def _fake_play(path): ) # And the temp file is cleaned up afterwards. assert not os.path.exists(played[0]), "temp WAV was not unlinked" + + +class TestPcmToWav: + """pcm_to_wav streams PCM through ffmpeg's stdin, not a temp file.""" + + def test_pcm_is_piped_to_stdin_not_staged_on_disk(self, tmp_path): + from plugins.platforms.discord.adapter import VoiceReceiver + + out = tmp_path / "out.wav" + with patch("plugins.platforms.discord.adapter.subprocess.run") as run: + VoiceReceiver.pcm_to_wav(b"\x00\x01" * 16, str(out)) + + args, kwargs = run.call_args + cmd = args[0] + assert kwargs["input"] == b"\x00\x01" * 16, "PCM must be fed via stdin" + assert "pipe:0" in cmd, "ffmpeg must read the PCM from stdin" + assert cmd[-1] == str(out), ( + "the WAV must be written to the real path; ffmpeg cannot seek on a " + "pipe, so a piped WAV gets placeholder RIFF/data sizes" + ) + assert not any(str(a).endswith(".pcm") for a in cmd), ( + "no temp .pcm file should be staged" + ) + + @pytest.mark.skipif( + __import__("shutil").which("ffmpeg") is None, reason="ffmpeg not installed", + ) + def test_output_wav_header_reports_true_length(self, tmp_path): + """A piped-stdout WAV reports 0xFFFFFFFF sizes; the written file must not.""" + import math + import struct + import wave + + from plugins.platforms.discord.adapter import VoiceReceiver + + frames = 48000 # 1s @ 48kHz stereo + pcm = b"".join( + struct.pack(" 16kHz is a 3x decimation of a 1s clip. + assert w.getnframes() == 16000 diff --git a/tests/gateway/test_weixin_secret_scope.py b/tests/gateway/test_weixin_secret_scope.py new file mode 100644 index 000000000000..8f5386949f99 --- /dev/null +++ b/tests/gateway/test_weixin_secret_scope.py @@ -0,0 +1,103 @@ +"""Weixin adapter secret-scope regression tests. + +The adapter's WEIXIN_* credential reads must follow the Slack pattern +(#59739): under multiplexing a SCOPED miss is authoritative (no borrow from +``os.environ`` — that would be a cross-profile leak), while an UNSCOPED read +(default-profile startup/send path) falls back to ``os.environ``, which is +that profile's own value, instead of raising ``UnscopedSecretError``. +""" + +import pytest + +from agent import secret_scope +from gateway.config import PlatformConfig +from gateway.platforms.weixin import WeixinAdapter, _wx_secret + + +@pytest.fixture() +def multiplex_on(): + previous = secret_scope.is_multiplex_active() + secret_scope.set_multiplex_active(True) + try: + yield + finally: + secret_scope.set_multiplex_active(previous) + + +class TestWxSecretHelper: + def test_scoped_read_uses_scope_value(self, multiplex_on, monkeypatch): + monkeypatch.setenv("WEIXIN_TOKEN", "default-profile-token") + token = secret_scope.set_secret_scope({"WEIXIN_TOKEN": "scoped-token"}) + try: + assert _wx_secret("WEIXIN_TOKEN") == "scoped-token" + finally: + secret_scope.reset_secret_scope(token) + + def test_scoped_miss_does_not_borrow_environ(self, multiplex_on, monkeypatch): + """A secondary profile without WEIXIN_TOKEN must NOT inherit the + default profile's process-env token.""" + monkeypatch.setenv("WEIXIN_TOKEN", "default-profile-token") + token = secret_scope.set_secret_scope({"OTHER_KEY": "x"}) + try: + assert _wx_secret("WEIXIN_TOKEN", "") == "" + assert _wx_secret("WEIXIN_TOKEN") is None + finally: + secret_scope.reset_secret_scope(token) + + def test_unscoped_read_falls_back_to_environ(self, multiplex_on, monkeypatch): + """The default profile's adapter runs unscoped under multiplexing; + os.environ is its own value — fall back instead of raising.""" + monkeypatch.setenv("WEIXIN_TOKEN", "default-profile-token") + token = secret_scope.set_secret_scope(None) + try: + assert _wx_secret("WEIXIN_TOKEN") == "default-profile-token" + finally: + secret_scope.reset_secret_scope(token) + + +class TestWeixinAdapterConstructionScope: + def test_multiplex_scoped_construction_reads_scope_not_environ( + self, multiplex_on, monkeypatch + ): + monkeypatch.setenv("WEIXIN_ACCOUNT_ID", "env-account") + monkeypatch.setenv("WEIXIN_TOKEN", "env-token") + token = secret_scope.set_secret_scope( + { + "WEIXIN_ACCOUNT_ID": "scoped-account", + "WEIXIN_TOKEN": "scoped-token", + } + ) + try: + adapter = WeixinAdapter(PlatformConfig(enabled=True)) + finally: + secret_scope.reset_secret_scope(token) + assert adapter._account_id == "scoped-account" + assert adapter._token == "scoped-token" + + def test_multiplex_scoped_miss_yields_empty_not_environ_borrow( + self, multiplex_on, monkeypatch + ): + monkeypatch.setenv("WEIXIN_ACCOUNT_ID", "env-account") + monkeypatch.setenv("WEIXIN_TOKEN", "env-token") + token = secret_scope.set_secret_scope({"SOMETHING_ELSE": "x"}) + try: + adapter = WeixinAdapter(PlatformConfig(enabled=True)) + finally: + secret_scope.reset_secret_scope(token) + assert adapter._account_id == "" + assert adapter._token == "" + + def test_multiplex_unscoped_construction_falls_back_to_environ( + self, multiplex_on, monkeypatch + ): + """Regression for the bare get_secret reads: default-profile adapter + construction under multiplexing must not raise UnscopedSecretError.""" + monkeypatch.setenv("WEIXIN_ACCOUNT_ID", "env-account") + monkeypatch.setenv("WEIXIN_TOKEN", "env-token") + token = secret_scope.set_secret_scope(None) + try: + adapter = WeixinAdapter(PlatformConfig(enabled=True)) + finally: + secret_scope.reset_secret_scope(token) + assert adapter._account_id == "env-account" + assert adapter._token == "env-token" diff --git a/tests/hermes_cli/test_gateway.py b/tests/hermes_cli/test_gateway.py index f6ffd3d51ffa..3d66de14b896 100644 --- a/tests/hermes_cli/test_gateway.py +++ b/tests/hermes_cli/test_gateway.py @@ -369,7 +369,7 @@ def test_kill_gateway_processes_force_uses_helper(self, monkeypatch): class TestStopProfileGateway: def test_stop_profile_gateway_keeps_pid_file_when_process_still_running(self, monkeypatch): - calls = {"kill": 0, "alive_probes": 0, "remove": 0} + calls = {"kill": 0, "alive_probes": 0, "remove": 0, "reap_calls": 0} monkeypatch.setattr("gateway.status.get_running_pid", lambda: 12345) # Post-#21561: the stop loop sends one SIGTERM via ``os.kill`` then @@ -389,11 +389,42 @@ def test_stop_profile_gateway_keeps_pid_file_when_process_still_running(self, mo "gateway.status.remove_pid_file", lambda: calls.__setitem__("remove", calls["remove"] + 1), ) + # Mock the orphan reap so it doesn't scan for real gateway processes + # (#75936 — stop_profile_gateway now calls _reap_unsupervised_gateway_orphans + # after killing the pid-file PID). + monkeypatch.setattr( + gateway, + "_reap_unsupervised_gateway_orphans", + lambda extra_exclude=None: calls.__setitem__("reap_calls", calls["reap_calls"] + 1) or False, + ) assert gateway.stop_profile_gateway() is True assert calls["kill"] == 1 # one SIGTERM assert calls["alive_probes"] == 20 # 20 liveness polls over the 2s window assert calls["remove"] == 0 + assert calls["reap_calls"] == 1 # orphan sweep ran after kill + + def test_stop_profile_gateway_excludes_killed_pid_from_orphan_reap(self, monkeypatch): + """The PID we killed must be excluded from the orphan sweep (#75936).""" + killed_pid = 99999 + reap_extra_excludes = [] + + monkeypatch.setattr("gateway.status.get_running_pid", lambda: killed_pid) + monkeypatch.setattr(gateway.os, "kill", lambda pid, sig: None) + monkeypatch.setattr("gateway.status._pid_exists", lambda pid: False) + monkeypatch.setattr("time.sleep", lambda _: None) + monkeypatch.setattr("gateway.status.remove_pid_file", lambda: None) + + def fake_reap(extra_exclude=None): + if extra_exclude: + reap_extra_excludes.append(extra_exclude) + return False + + monkeypatch.setattr(gateway, "_reap_unsupervised_gateway_orphans", fake_reap) + + assert gateway.stop_profile_gateway() is True + assert len(reap_extra_excludes) == 1 + assert killed_pid in reap_extra_excludes[0] def test_module_has_logger(): diff --git a/tests/hermes_cli/test_mem_trim.py b/tests/hermes_cli/test_mem_trim.py new file mode 100644 index 000000000000..3c94f23f07eb --- /dev/null +++ b/tests/hermes_cli/test_mem_trim.py @@ -0,0 +1,216 @@ +"""Tests for the long-lived gateway heap-trim helper.""" + +from unittest.mock import Mock + +import pytest + +import hermes_cli.mem_trim as mem_trim + + +@pytest.fixture(autouse=True) +def _reset_trim_state(monkeypatch): + monkeypatch.setattr(mem_trim, "_last_trim_monotonic", 0.0) + monkeypatch.setattr(mem_trim, "_probe_done", True) + monkeypatch.setattr(mem_trim, "_malloc_trim", None) + monkeypatch.setattr(mem_trim, "_trim_call_count", 0) + + +def test_unsupported_allocator_is_noop_without_gc(monkeypatch): + collect = Mock() + monkeypatch.setattr(mem_trim.gc, "collect", collect) + + assert mem_trim.trim_memory(force=True, reason="test") is False + collect.assert_not_called() + + +def test_config_kill_switch_overrides_force_from_config_file(monkeypatch, tmp_path): + from hermes_constants import reset_hermes_home_override, set_hermes_home_override + + hermes_home = tmp_path / "hermes" + hermes_home.mkdir() + (hermes_home / "config.yaml").write_text( + "context:\n memory_trim:\n enabled: false\n", + encoding="utf-8", + ) + trim = Mock(return_value=1) + monkeypatch.setattr(mem_trim, "_malloc_trim", trim) + token = set_hermes_home_override(hermes_home) + + try: + assert mem_trim.trim_memory(force=True) is False + trim.assert_not_called() + finally: + reset_hermes_home_override(token) + + +def test_default_config_declares_memory_trim_controls(): + from hermes_cli.config import DEFAULT_CONFIG + + context = DEFAULT_CONFIG["context"] + assert isinstance(context, dict) + settings = context["memory_trim"] + assert isinstance(settings, dict) + assert isinstance(settings["enabled"], bool) + assert isinstance(settings["cooldown_seconds"], float) + + +def test_collect_memory_snapshot_parses_linux_proc_status(monkeypatch): + monkeypatch.setattr(mem_trim.sys, "platform", "linux") + monkeypatch.setattr( + mem_trim, + "_read_proc_status", + lambda: "Name:\tpython\nVmRSS:\t1234 kB\nRssAnon:\t567 kB\n", + ) + monkeypatch.setattr(mem_trim.threading, "active_count", lambda: 9) + + assert mem_trim.collect_memory_snapshot(history_bytes=42) == { + "rss_kib": 1234, + "rss_anon_kib": 567, + "thread_count": 9, + "history_bytes": 42, + } + + +def test_success_collects_then_trims(monkeypatch): + calls = [] + monkeypatch.setattr(mem_trim.gc, "collect", lambda: calls.append("gc")) + monkeypatch.setattr( + mem_trim, "_malloc_trim", lambda pad: calls.append(("trim", pad)) or 1 + ) + monkeypatch.setattr(mem_trim.time, "monotonic", lambda: 100.0) + + assert mem_trim.trim_memory(reason="turn", cooldown_seconds=60) is True + assert calls == ["gc", ("trim", 0)] + assert mem_trim._last_trim_monotonic == 100.0 + + +def test_success_logs_memory_snapshot_and_trim_result(monkeypatch, caplog): + monkeypatch.setattr(mem_trim.gc, "collect", lambda: None) + monkeypatch.setattr(mem_trim, "_malloc_trim", lambda _pad: 1) + monkeypatch.setattr(mem_trim.time, "monotonic", lambda: 100.0) + snapshots = iter( + ( + {"rss_kib": 4096, "rss_anon_kib": 3072, "thread_count": 3}, + {"rss_kib": 2048, "rss_anon_kib": 1024, "thread_count": 3}, + ) + ) + monkeypatch.setattr(mem_trim, "collect_memory_snapshot", lambda: next(snapshots)) + + with caplog.at_level("INFO", logger="hermes_cli.mem_trim"): + assert mem_trim.trim_memory(reason="test turn") is True + + assert "reason=test turn" in caplog.text + assert "malloc_trim=1" in caplog.text + assert "rss_kib=4096->2048" in caplog.text + + +def test_force_logs_even_when_periodic_log_sampling_skips(monkeypatch, caplog): + monkeypatch.setattr(mem_trim.gc, "collect", lambda: None) + monkeypatch.setattr(mem_trim, "_malloc_trim", lambda _pad: 1) + monkeypatch.setattr(mem_trim, "_config_settings", lambda: (True, 0.0, 99, 1.0)) + # Two ticks: the forced call comes after the 5s force floor so it runs + # (the floor exists to coalesce burst closes, not to mute logging). + _ticks = iter([100.0, 110.0]) + monkeypatch.setattr(mem_trim.time, "monotonic", lambda: next(_ticks, 110.0)) + monkeypatch.setattr( + mem_trim, + "collect_memory_snapshot", + lambda: {"rss_kib": 4096, "rss_anon_kib": 3072, "thread_count": 3}, + ) + + with caplog.at_level("INFO", logger="hermes_cli.mem_trim"): + assert mem_trim.trim_memory(reason="periodic") is True + assert mem_trim.trim_memory(force=True, reason="close") is True + + messages = [record.getMessage() for record in caplog.records] + assert not any("reason=periodic" in message for message in messages) + assert any("reason=close" in message for message in messages) + + +def test_cooldown_suppresses_repeated_collection(monkeypatch): + collect = Mock() + trim = Mock(return_value=1) + monkeypatch.setattr(mem_trim.gc, "collect", collect) + monkeypatch.setattr(mem_trim, "_malloc_trim", trim) + monkeypatch.setattr(mem_trim, "_last_trim_monotonic", 95.0) + monkeypatch.setattr(mem_trim.time, "monotonic", lambda: 100.0) + + assert mem_trim.trim_memory(cooldown_seconds=60) is False + collect.assert_not_called() + trim.assert_not_called() + assert mem_trim.trim_memory(force=True, cooldown_seconds=60) is True + + +def test_config_cooldown_controls_rate_limit(monkeypatch): + trim = Mock(return_value=1) + monkeypatch.setattr(mem_trim, "_malloc_trim", trim) + monkeypatch.setattr(mem_trim, "_last_trim_monotonic", 1.0) + monkeypatch.setattr(mem_trim.time, "monotonic", lambda: 100.0) + monkeypatch.setattr( + "hermes_cli.config.load_config_readonly", + lambda: { + "context": { + "memory_trim": {"enabled": True, "cooldown_seconds": 120.0} + } + }, + ) + + assert mem_trim.trim_memory() is False + trim.assert_not_called() + + +def test_legacy_environment_switch_does_not_control_behavior(monkeypatch): + trim = Mock(return_value=1) + monkeypatch.setattr(mem_trim, "_malloc_trim", trim) + monkeypatch.setenv("HERMES_DISABLE_MEMORY_TRIM", "1") + monkeypatch.setattr( + "hermes_cli.config.load_config_readonly", + lambda: {"context": {"memory_trim": {"enabled": True}}}, + ) + + assert mem_trim.trim_memory(force=True) is True + trim.assert_called_once_with(0) + + +def test_libc_failure_is_fail_open_and_rate_limited(monkeypatch): + trim = Mock(side_effect=RuntimeError("boom")) + monkeypatch.setattr(mem_trim, "_malloc_trim", trim) + monkeypatch.setattr(mem_trim.time, "monotonic", lambda: 100.0) + + assert mem_trim.trim_memory(reason="test", cooldown_seconds=60) is False + assert mem_trim._last_trim_monotonic == 100.0 + assert mem_trim.trim_memory(cooldown_seconds=60) is False + assert trim.call_count == 1 + + +def test_force_floor_coalesces_burst_closes(monkeypatch): + """A delegate batch closes N child agents back-to-back, each forcing a + trim — the short force floor must coalesce the burst instead of stacking + N uncooled full gc.collect() passes in the same process.""" + collect = Mock() + trim = Mock(return_value=1) + monkeypatch.setattr(mem_trim.gc, "collect", collect) + monkeypatch.setattr(mem_trim, "_malloc_trim", trim) + monkeypatch.setattr(mem_trim, "_config_settings", lambda: (True, 0.0, 1, 0.0)) + monkeypatch.setattr( + mem_trim, + "collect_memory_snapshot", + lambda: {"rss_kib": 4096, "rss_anon_kib": 3072, "thread_count": 3}, + ) + monkeypatch.setattr(mem_trim, "_last_trim_monotonic", 0.0) + + # t=100: first forced close runs. + monkeypatch.setattr(mem_trim.time, "monotonic", lambda: 100.0) + assert mem_trim.trim_memory(force=True, reason="agent close") is True + assert trim.call_count == 1 + + # t=101..103: three more child closes inside the floor — all coalesced. + for t in (101.0, 102.0, 103.0): + monkeypatch.setattr(mem_trim.time, "monotonic", lambda t=t: t) + assert mem_trim.trim_memory(force=True, reason="agent close") is False + assert trim.call_count == 1, "burst closes must not stack forced trims" + + # t=106: past the floor — the parent's final close-trim still fires. + monkeypatch.setattr(mem_trim.time, "monotonic", lambda: 106.0) + assert mem_trim.trim_memory(force=True, reason="agent close") is True + assert trim.call_count == 2 diff --git a/tests/hermes_cli/test_model_switch_custom_providers.py b/tests/hermes_cli/test_model_switch_custom_providers.py index e1eef0bc5a39..5b3513dc82dc 100644 --- a/tests/hermes_cli/test_model_switch_custom_providers.py +++ b/tests/hermes_cli/test_model_switch_custom_providers.py @@ -640,7 +640,7 @@ def fake_fetch_api_models(api_key, base_url, **kwargs): assert gateway_prov is not None, "Custom provider group not found in results" assert calls == [ - ("sk-gateway-key", "https://gateway.example.com/v1", {"headers": None}) + ("sk-gateway-key", "https://gateway.example.com/v1", {"timeout": 5.0, "headers": None}) ], "fetch_api_models must be called with the custom provider's credentials" assert gateway_prov["models"] == [ "gateway-model-a", diff --git a/tests/hermes_cli/test_nous_portal_staging_allowlist.py b/tests/hermes_cli/test_nous_portal_staging_allowlist.py index 71cf4a4981fa..34e02049dd9b 100644 --- a/tests/hermes_cli/test_nous_portal_staging_allowlist.py +++ b/tests/hermes_cli/test_nous_portal_staging_allowlist.py @@ -87,6 +87,11 @@ def _write_auth_file(self, tmp_path, *, stored_portal_url): def _run_and_capture(self, monkeypatch, auth): seen_portal_urls = [] + # The resolve memo is module-level state; clear it so each test's + # resolution actually exercises the refresh path instead of serving + # a token cached by a previous test. + monkeypatch.setattr(auth, "_RESOLVE_TOKEN_CACHE", None) + def _fake_refresh(*, client, portal_base_url, client_id, refresh_token): seen_portal_urls.append(portal_base_url) return { diff --git a/tests/hermes_cli/test_resolve_token_memo.py b/tests/hermes_cli/test_resolve_token_memo.py new file mode 100644 index 000000000000..8fabb8d64c52 --- /dev/null +++ b/tests/hermes_cli/test_resolve_token_memo.py @@ -0,0 +1,96 @@ +"""Tests for the resolve_nous_access_token startup-burst memo (PR #66016). + +The memo collapses the startup burst of managed-tool check_fn calls into a +single expensive resolution: within the short TTL, repeat calls return the +cached token without re-entering _provider_state_transaction (two +cross-process file locks + state reads) or triggering a network refresh. +""" + +import json +import time + +import pytest + +import hermes_cli.auth as auth + + +@pytest.fixture(autouse=True) +def _fresh_memo(monkeypatch, tmp_path): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.delenv("HERMES_PORTAL_BASE_URL", raising=False) + monkeypatch.delenv("NOUS_PORTAL_BASE_URL", raising=False) + monkeypatch.setattr(auth, "_RESOLVE_TOKEN_CACHE", None) + yield + + +def _write_valid_auth_file(tmp_path, token="memo-token"): + (tmp_path / "auth.json").write_text( + json.dumps( + { + "version": 1, + "active_provider": "nous", + "providers": { + "nous": { + "access_token": token, + "refresh_token": "r", + "client_id": "hermes-cli-vps", + "expires_at": time.strftime( + "%Y-%m-%dT%H:%M:%S+00:00", time.gmtime(time.time() + 3600) + ), + } + }, + } + ) + ) + + +def _count_transactions(monkeypatch): + calls = {"n": 0} + real = auth._provider_state_transaction + + def _counting(provider): + calls["n"] += 1 + return real(provider) + + monkeypatch.setattr(auth, "_provider_state_transaction", _counting) + return calls + + +def test_repeat_calls_within_ttl_hit_memo(monkeypatch, tmp_path): + _write_valid_auth_file(tmp_path) + calls = _count_transactions(monkeypatch) + + first = auth.resolve_nous_access_token() + second = auth.resolve_nous_access_token() + third = auth.resolve_nous_access_token() + + assert first == second == third == "memo-token" + assert calls["n"] == 1, ( + "repeat calls within the TTL must not re-enter the state transaction" + ) + + +def test_memo_expires_after_ttl(monkeypatch, tmp_path): + _write_valid_auth_file(tmp_path) + calls = _count_transactions(monkeypatch) + + auth.resolve_nous_access_token() + cached_at, tok = auth._RESOLVE_TOKEN_CACHE + monkeypatch.setattr( + auth, + "_RESOLVE_TOKEN_CACHE", + (cached_at - auth._RESOLVE_TOKEN_CACHE_TTL_S - 1.0, tok), + ) + auth.resolve_nous_access_token() + + assert calls["n"] == 2, "an expired memo must re-resolve" + + +def test_insecure_callers_bypass_memo(monkeypatch, tmp_path): + _write_valid_auth_file(tmp_path) + calls = _count_transactions(monkeypatch) + + auth.resolve_nous_access_token() + auth.resolve_nous_access_token(insecure=True) + + assert calls["n"] == 2, "insecure callers must bypass the memo entirely" diff --git a/tests/hermes_cli/test_update_eol_churn.py b/tests/hermes_cli/test_update_eol_churn.py index fb742d8001dd..a8fe2f762f51 100644 --- a/tests/hermes_cli/test_update_eol_churn.py +++ b/tests/hermes_cli/test_update_eol_churn.py @@ -59,6 +59,19 @@ def _managed_repo(tmp_path: Path, files: dict[str, bytes]) -> Path: for name in files: (repo / name).unlink() _git(repo, "checkout", "--", ".") + # Deterministic dirtiness: whether `git diff` content-checks an entry (and + # so sees the CRLF churn) or trusts the stat cache depends on racy-git + # detection — entries whose mtime equals the index timestamp get content- + # compared, later ones read clean. On a fast runner a large checkout + # straddles that boundary nondeterministically (CI flake: 92/661 of 1200 + # dirty). Bump every worktree mtime past the index write so ALL entries + # are stat-stale and git must content-compare each one. + import os as _os + import time as _time + + bumped = _time.time() + 5 + for name in files: + _os.utime(repo / name, (bumped, bumped)) return repo diff --git a/tests/hermes_cli/test_user_providers_model_switch.py b/tests/hermes_cli/test_user_providers_model_switch.py index 46df5c0baeba..36c1d6d13cf4 100644 --- a/tests/hermes_cli/test_user_providers_model_switch.py +++ b/tests/hermes_cli/test_user_providers_model_switch.py @@ -158,7 +158,7 @@ def fake_fetch_api_models(api_key, base_url, **kwargs): ) assert user_prov is not None - assert calls == [("sk-test", "http://127.0.0.1:3000/api/v1", {"headers": None})] + assert calls == [("sk-test", "http://127.0.0.1:3000/api/v1", {"timeout": 5.0, "headers": None})] assert user_prov["models"] == ["old-configured-model", "new-live-model"] assert user_prov["total_models"] == 2 @@ -497,7 +497,7 @@ def _fake_fetch(api_key, api_url, **kwargs): assert probed.get("called") is True, "no-key bare endpoint should be probed" assert probed["api_key"] == "" - assert probed["kwargs"] == {"headers": None} + assert probed["kwargs"] == {"timeout": 5.0, "headers": None} row = next(p for p in providers if p["slug"] == "local-llamacpp") assert row["models"] == ["live-model-1", "live-model-2", "live-model-3"] assert row["total_models"] == 3 diff --git a/tests/hermes_cli/test_web_server.py b/tests/hermes_cli/test_web_server.py index 2a201966cab4..c7f3c5fc85da 100644 --- a/tests/hermes_cli/test_web_server.py +++ b/tests/hermes_cli/test_web_server.py @@ -318,6 +318,39 @@ def test_get_sessions_poll_preserves_pending_wal(self): monitor.close() writer.close() + def test_get_status_loads_gateway_config_off_event_loop(self, monkeypatch): + """Cold gateway config loading must not block the WebSocket loop. + + On Windows the first ``load_gateway_config()`` call imports and + discovers platform adapters and can take longer than Desktop's 15s + WebSocket timeout. Running it inline makes a concurrent /api/ws + handshake time out before ``gateway.ready`` can be sent. + """ + import gateway.config as gateway_config + import hermes_cli.web_server as web_server + + seen = {} + + class _Config: + @staticmethod + def get_connected_platforms(): + return [] + + def _load(): + seen["thread"] = threading.get_ident() + return _Config() + + monkeypatch.setattr(gateway_config, "load_gateway_config", _load) + + async def _run(): + event_loop_thread = threading.get_ident() + await web_server.get_status() + return event_loop_thread + + event_loop_thread = asyncio.run(_run()) + + assert seen["thread"] != event_loop_thread + def test_get_sessions_auto_archive_uses_maintenance_writer(self): from hermes_cli import web_server from hermes_cli.config import load_config, save_config diff --git a/tests/plugins/memory/test_hindsight_provider.py b/tests/plugins/memory/test_hindsight_provider.py index 0c72bd6203df..e3ba1b8a55ba 100644 --- a/tests/plugins/memory/test_hindsight_provider.py +++ b/tests/plugins/memory/test_hindsight_provider.py @@ -10,6 +10,7 @@ import re import stat import sys +import time from pathlib import Path from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock @@ -501,6 +502,209 @@ def test_queue_prefetch_skipped_in_tools_mode(self, provider_with_config): # Should not start a thread assert p._prefetch_thread is None + def test_prefetch_waits_for_pending_retain_before_recall(self, provider): + """The background prefetch must wait for queued retains to drain so the + next turn's recall observes the just-completed turn (no retain race).""" + import threading + + order = [] + release = threading.Event() + + async def _slow_retain(*args, **kwargs): + release.wait(timeout=5.0) + order.append("retain") + + async def _recall(**kwargs): + order.append("recall") + return SimpleNamespace(results=[SimpleNamespace(text="m")]) + + provider._client.aretain_batch = AsyncMock(side_effect=_slow_retain) + provider._client.arecall = AsyncMock(side_effect=_recall) + + # Enqueue a slow retain, then immediately queue the next-turn prefetch. + provider.sync_turn("hello", "world") + provider.queue_prefetch("next turn query") + + # Let the prefetch thread start and reach the drain barrier. + time.sleep(0.2) + assert order == [], "recall ran before the pending retain drained" + + # Release the retain; the prefetch should now proceed AFTER it. + release.set() + if provider._prefetch_thread: + provider._prefetch_thread.join(timeout=5.0) + provider._retain_queue.join() + assert order and order[0] == "retain" + assert "recall" in order + + def test_prefetch_wait_for_retain_can_be_disabled(self, provider_with_config): + p = provider_with_config(prefetch_waits_for_retain=False) + p._client = _make_mock_client() + assert p._prefetch_waits_for_retain is False + + +class TestPrefetchServerRetainVisibility: + """PR #62871 review follow-up: draining the local writer queue is not a + read-after-write signal for async retains. With ``retain_async=True`` the + server accepts the write and returns an ``operation_id`` that stays + ``pending`` until the write is durable/recall-visible. The background + prefetch must gate on server-side operation completion, not just the local + queue, before recalling. + """ + + def _client_with_ops(self, statuses): + """Mock client whose aretain_batch returns an async operation_id and + whose operations.get_operation_status yields *statuses* in order + (last value repeats).""" + client = _make_mock_client() + client.aretain_batch = AsyncMock( + return_value=SimpleNamespace(operation_id="op-1", operation_ids=None) + ) + seq = list(statuses) + + async def _status(**kwargs): + value = seq.pop(0) if len(seq) > 1 else seq[0] + return SimpleNamespace(status=value) + + client.operations = MagicMock() + client.operations.get_operation_status = AsyncMock(side_effect=_status) + return client + + def test_tracks_async_operation_id_from_retain(self, provider): + provider._client.aretain_batch = AsyncMock( + return_value=SimpleNamespace(operation_id="op-async-1", operation_ids=None) + ) + provider.sync_turn("hello", "world") + provider._retain_queue.join() + assert "op-async-1" in provider._pending_retain_ops + + def test_tracks_multiple_operation_ids(self, provider): + provider._client.aretain_batch = AsyncMock( + return_value=SimpleNamespace( + operation_id=None, operation_ids=["op-a", "op-b"] + ) + ) + provider.sync_turn("hello", "world") + provider._retain_queue.join() + assert {"op-a", "op-b"} <= provider._pending_retain_ops + + def test_sync_retain_tracks_no_ops(self, provider_with_config): + p = provider_with_config(retain_async=False) + p._client = _make_mock_client() + p._client.aretain_batch = AsyncMock( + return_value=SimpleNamespace(operation_id="op-x", operation_ids=None) + ) + p.sync_turn("hello", "world") + p._retain_queue.join() + # retain_async=False → no server-side op to wait on. + assert p._pending_retain_ops == set() + + def test_prefetch_waits_for_server_completion_before_recall(self, provider): + """Recall must not run until the tracked async op reports completed.""" + order = [] + + async def _recall(**kwargs): + order.append("recall") + return SimpleNamespace(results=[SimpleNamespace(text="m")]) + + provider._client = self._client_with_ops(["pending", "pending", "completed"]) + provider._client.arecall = AsyncMock(side_effect=_recall) + + provider.sync_turn("hello", "world") + provider._retain_queue.join() + assert "op-1" in provider._pending_retain_ops + + provider.queue_prefetch("next turn query") + if provider._prefetch_thread: + provider._prefetch_thread.join(timeout=5.0) + + # Recall ran, the op was polled to completion, and the pending set + # was cleared (so a later prefetch won't re-poll it). + assert order == ["recall"] + assert provider._client.operations.get_operation_status.await_count >= 3 + assert provider._pending_retain_ops == set() + + def test_prefetch_proceeds_after_server_wait_timeout(self, provider_with_config): + """A wedged/never-completing async op must not hang prefetch forever; + it recalls anyway once the drain budget is exhausted.""" + p = provider_with_config(prefetch_retain_drain_timeout=0.3) + order = [] + + async def _recall(**kwargs): + order.append("recall") + return SimpleNamespace(results=[SimpleNamespace(text="m")]) + + p._client = self._client_with_ops(["pending"]) # never completes + p._client.arecall = AsyncMock(side_effect=_recall) + + p.sync_turn("hello", "world") + p._retain_queue.join() + + start = time.monotonic() + p.queue_prefetch("next turn query") + if p._prefetch_thread: + p._prefetch_thread.join(timeout=5.0) + elapsed = time.monotonic() - start + + assert order == ["recall"], "prefetch should recall after the timeout" + assert elapsed < 3.0, "prefetch must not block well past the drain budget" + + def test_timed_out_ops_are_dropped_not_repolled(self, provider_with_config): + """Ops unresolved at deadline must be EVICTED so a permanently failing + status endpoint can't make every later prefetch re-burn the full + timeout on a growing pending set (unbounded session-wide degradation + + reply-path join penalty).""" + p = provider_with_config(prefetch_retain_drain_timeout=0.3) + p._client = self._client_with_ops(["pending"]) # never completes + p._client.arecall = AsyncMock( + return_value=SimpleNamespace(results=[SimpleNamespace(text="m")]) + ) + + p.sync_turn("hello", "world") + p._retain_queue.join() + assert p._pending_retain_ops, "op should be tracked before the wait" + + # First prefetch burns the budget and must DROP the wedged op. + p.queue_prefetch("q1") + if p._prefetch_thread: + p._prefetch_thread.join(timeout=5.0) + assert p._pending_retain_ops == set(), ( + "unresolved ops must be evicted at deadline, not retained" + ) + + # A later prefetch with nothing pending must be near-instant. + start = time.monotonic() + p.queue_prefetch("q2") + if p._prefetch_thread: + p._prefetch_thread.join(timeout=5.0) + assert time.monotonic() - start < 0.25, ( + "second prefetch re-polled dropped ops — eviction regressed" + ) + + def test_operation_notfound_treated_as_complete(self, provider): + """A NotFound (completed+evicted) op is treated as done, not pending.""" + from hindsight_client_api.exceptions import NotFoundException + + client = _make_mock_client() + client.operations = MagicMock() + client.operations.get_operation_status = AsyncMock( + side_effect=NotFoundException(status=404, reason="gone") + ) + provider._client = client + + assert provider._is_retain_op_complete("bank", "op-gone") is True + + def test_transient_status_error_keeps_waiting(self, provider): + """A transient status-check error means 'unknown', so keep waiting.""" + client = _make_mock_client() + client.operations = MagicMock() + client.operations.get_operation_status = AsyncMock( + side_effect=RuntimeError("temporary blip") + ) + provider._client = client + + assert provider._is_retain_op_complete("bank", "op-1") is False + # --------------------------------------------------------------------------- # sync_turn tests diff --git a/tests/plugins/test_holographic_vector_storage.py b/tests/plugins/test_holographic_vector_storage.py new file mode 100644 index 000000000000..70e984ab25f8 --- /dev/null +++ b/tests/plugins/test_holographic_vector_storage.py @@ -0,0 +1,205 @@ +"""Storage-size regression tests for holographic HRR vectors.""" + +from __future__ import annotations + +import pytest + +np = pytest.importorskip("numpy") + +from plugins.memory.holographic import holographic as hrr +from plugins.memory.holographic.retrieval import FactRetriever +from plugins.memory.holographic.store import MemoryStore + + +pytestmark = pytest.mark.skipif( + not hrr._HAS_NUMPY, + reason="holographic vector storage requires numpy", +) + + +def _float32_blob_size(dim: int) -> int: + return len(hrr._FLOAT32_BLOB_PREFIX) + dim * np.dtype(np.float32).itemsize + + +def test_phases_to_bytes_stores_float32_and_round_trips_with_dim() -> None: + dim = 1024 + phases = hrr.encode_atom("storage-size-regression", dim=dim) + + blob = hrr.phases_to_bytes(phases) + + assert len(blob) == _float32_blob_size(dim) + restored = hrr.bytes_to_phases(blob, dim=dim) + assert restored.shape == (dim,) + np.testing.assert_allclose(restored, phases, rtol=0, atol=1e-6) + + +def test_phases_to_bytes_round_trips_without_dim() -> None: + dim = 1024 + phases = hrr.encode_atom("dimensionless-round-trip", dim=dim) + + restored = hrr.bytes_to_phases(hrr.phases_to_bytes(phases)) + + assert restored.shape == (dim,) + np.testing.assert_allclose(restored, phases, rtol=0, atol=1e-6) + + +def test_phases_to_bytes_round_trips_ambiguous_small_dims_without_dim() -> None: + dim = 2 + phases = hrr.encode_atom("ambiguous-small-dimension", dim=dim) + + restored = hrr.bytes_to_phases(hrr.phases_to_bytes(phases)) + + assert restored.shape == (dim,) + np.testing.assert_allclose(restored, phases, rtol=0, atol=1e-6) + + +def test_bytes_to_phases_rejects_malformed_float32_blobs() -> None: + phases = hrr.encode_atom("malformed-float32-blob", dim=2) + blob = hrr.phases_to_bytes(phases) + + with pytest.raises(ValueError, match="expected .* for dim=3"): + hrr.bytes_to_phases(blob, dim=3) + + with pytest.raises(ValueError, match="invalid payload byte length"): + hrr.bytes_to_phases(hrr._FLOAT32_BLOB_PREFIX + b"x") + + +def test_bytes_to_phases_reads_legacy_float64_blobs_with_and_without_dim() -> None: + dim = 1024 + phases = hrr.encode_atom("legacy-float64-regression", dim=dim) + legacy_blob = phases.astype(np.float64, copy=False).tobytes() + + assert len(legacy_blob) == dim * np.dtype(np.float64).itemsize + restored_with_dim = hrr.bytes_to_phases(legacy_blob, dim=dim) + restored_without_dim = hrr.bytes_to_phases(legacy_blob) + + assert restored_with_dim.shape == (dim,) + assert restored_without_dim.shape == (dim,) + np.testing.assert_allclose(restored_with_dim, phases, rtol=0, atol=0) + np.testing.assert_allclose(restored_without_dim, phases, rtol=0, atol=0) + + +def test_bytes_to_phases_prefers_dim_matched_legacy_float64_on_prefix_collision() -> None: + dim = 4 + legacy_blob = hrr._FLOAT32_BLOB_PREFIX + b"\0" * ( + dim * np.dtype(np.float64).itemsize - len(hrr._FLOAT32_BLOB_PREFIX) + ) + + restored = hrr.bytes_to_phases(legacy_blob, dim=dim) + + assert restored.shape == (dim,) + np.testing.assert_array_equal( + restored, + np.frombuffer(legacy_blob, dtype=np.float64).copy(), + ) + + +def test_dim1_phases_to_bytes_writes_legacy_float64() -> None: + """At dim=1 the float32 prefixed blob (8 B) collides with raw float64 + (8 B), so phases_to_bytes must fall back to raw float64.""" + dim = 1 + phases = hrr.encode_atom("dim-one-ambiguity", dim=dim) + + blob = hrr.phases_to_bytes(phases, dim=dim) + + assert len(blob) == dim * np.dtype(np.float64).itemsize # 8 bytes, no prefix + assert not blob.startswith(hrr._FLOAT32_BLOB_PREFIX) + + +def test_dim1_round_trip_with_dim() -> None: + """Round-trip at dim=1 must work via the legacy float64 path.""" + dim = 1 + phases = hrr.encode_atom("dim-one-round-trip", dim=dim) + + restored = hrr.bytes_to_phases(hrr.phases_to_bytes(phases, dim=dim), dim=dim) + + assert restored.shape == (dim,) + np.testing.assert_allclose(restored, phases, rtol=0, atol=0) + + +def test_dim1_legacy_blob_starting_with_prefix_decodes_as_float64() -> None: + """A legacy float64 blob at dim=1 that happens to start with HRR1 must + decode as float64, not be misread as a prefixed float32 blob.""" + dim = 1 + phases = hrr.encode_atom("prefix-collision-dim-one", dim=dim) + legacy_blob = phases.astype(np.float64).tobytes() + # Force the blob to start with HRR1 prefix bytes + collision_blob = hrr._FLOAT32_BLOB_PREFIX + legacy_blob[len(hrr._FLOAT32_BLOB_PREFIX):] + assert len(collision_blob) == dim * np.dtype(np.float64).itemsize + + restored = hrr.bytes_to_phases(collision_blob, dim=dim) + + assert restored.shape == (dim,) + np.testing.assert_allclose(restored, np.frombuffer(collision_blob, dtype=np.float64).copy(), rtol=0, atol=0) + + +def test_memory_store_reads_legacy_float64_vectors(tmp_path) -> None: + dim = 64 + db_path = tmp_path / "legacy_memory_store.db" + + with MemoryStore(db_path=db_path, hrr_dim=dim) as store: + fact_id = store.add_fact( + 'Bob Stone keeps "legacy HRR vectors" searchable.', + category="compat", + tags="legacy storage", + ) + + fact_blob = store._conn.execute( + "SELECT hrr_vector FROM facts WHERE fact_id = ?", + (fact_id,), + ).fetchone()["hrr_vector"] + bank_blob = store._conn.execute( + "SELECT vector FROM memory_banks WHERE bank_name = ?", + ("cat:compat",), + ).fetchone()["vector"] + + legacy_fact_blob = hrr.bytes_to_phases(fact_blob, dim=dim).astype(np.float64).tobytes() + legacy_bank_blob = hrr.bytes_to_phases(bank_blob, dim=dim).astype(np.float64).tobytes() + store._conn.execute( + "UPDATE facts SET hrr_vector = ? WHERE fact_id = ?", + (legacy_fact_blob, fact_id), + ) + store._conn.execute( + "UPDATE memory_banks SET vector = ? WHERE bank_name = ?", + (legacy_bank_blob, "cat:compat"), + ) + store._conn.commit() + + assert len(legacy_fact_blob) == dim * np.dtype(np.float64).itemsize + assert len(legacy_bank_blob) == dim * np.dtype(np.float64).itemsize + + retriever = FactRetriever(store, hrr_dim=dim) + results = retriever.search("legacy HRR vectors", category="compat", limit=1) + + assert results + assert results[0]["fact_id"] == fact_id + + +def test_memory_store_persists_fact_and_bank_vectors_as_float32(tmp_path) -> None: + dim = 64 + db_path = tmp_path / "memory_store.db" + + with MemoryStore(db_path=db_path, hrr_dim=dim) as store: + fact_id = store.add_fact( + 'Alice Smith stores "compact HRR vectors" for Python tests.', + category="perf", + tags="hrr storage", + ) + + fact_blob = store._conn.execute( + "SELECT hrr_vector FROM facts WHERE fact_id = ?", + (fact_id,), + ).fetchone()["hrr_vector"] + bank_blob = store._conn.execute( + "SELECT vector FROM memory_banks WHERE bank_name = ?", + ("cat:perf",), + ).fetchone()["vector"] + + assert len(fact_blob) == _float32_blob_size(dim) + assert len(bank_blob) == _float32_blob_size(dim) + + retriever = FactRetriever(store, hrr_dim=dim) + results = retriever.search("compact HRR vectors", category="perf", limit=1) + + assert results + assert results[0]["fact_id"] == fact_id diff --git a/tests/run_agent/test_interrupt_propagation.py b/tests/run_agent/test_interrupt_propagation.py index 7e3085f1d2c0..f53afe71864c 100644 --- a/tests/run_agent/test_interrupt_propagation.py +++ b/tests/run_agent/test_interrupt_propagation.py @@ -27,6 +27,7 @@ def _make_bare_agent(self): agent = AIAgent.__new__(AIAgent) agent._interrupt_requested = False agent._interrupt_message = None + agent._hard_interrupt_requested = threading.Event() agent._execution_thread_id = None agent._interrupt_thread_signal_pending = False agent._active_children = [] @@ -54,6 +55,38 @@ def test_parent_interrupt_sets_child_flag(self): assert is_interrupted() is False assert parent._interrupt_thread_signal_pending is True + def test_hard_cancel_is_explicit_atomic_and_propagated(self): + parent = self._make_bare_agent() + child = self._make_bare_agent() + parent._active_children.append(child) + + parent.interrupt("Stop requested", hard_cancel=True) + + assert parent._hard_interrupt_requested.is_set() + assert child._hard_interrupt_requested.is_set() + parent.clear_interrupt() + assert not parent._hard_interrupt_requested.is_set() + + def test_message_interrupt_does_not_set_hard_cancel(self): + agent = self._make_bare_agent() + + agent.interrupt("new user message") + + assert agent._interrupt_requested is True + assert not agent._hard_interrupt_requested.is_set() + + def test_active_turn_redirect_does_not_set_hard_cancel(self): + agent = self._make_bare_agent() + agent._model_request_active = threading.Event() + agent._model_request_active.set() + agent._pending_redirect = None + + assert agent.redirect("new correction") is True + + assert agent._interrupt_requested is True + assert agent._interrupt_message is None + assert not agent._hard_interrupt_requested.is_set() + def test_child_clear_interrupt_at_start_clears_thread(self): """child.clear_interrupt() at start of run_conversation clears the bound execution thread's interrupt flag. diff --git a/tests/test_secret_scope_plugin_families.py b/tests/test_secret_scope_plugin_families.py new file mode 100644 index 000000000000..25d093b7185a --- /dev/null +++ b/tests/test_secret_scope_plugin_families.py @@ -0,0 +1,260 @@ +"""Regression tests: plugin-family credential reads honor the profile secret scope. + +Class-closure follow-up to the profile secret-scope cluster (#76462). Memory, +image_gen, and browser plugins, plus a handful of tier-3 tool helpers, read +credentials straight from ``os.environ``. Under a multiplexed gateway the +process environment may hold ANOTHER profile's key (or none), so every +credential read must route through ``agent.secret_scope.get_secret`` and honor +its verdict — a scoped miss under multiplexing returns the default and must +NOT borrow from ``os.environ``. + +One representative test pair (scoped-wins / scoped-miss-no-borrow) per plugin +family, plus the two behavioral sites: + +* supermemory ``post_setup`` must not write a profile's key into the + process-global environ when multiplexing is active (sibling-profile + pollution). +* google_meet ``process_manager.start`` must resolve OPENAI_API_KEY through + the scope AT SPAWN TIME and pass it explicitly in the child environment — + the detached child inherits the process env, not the contextvar scope. +""" + +from __future__ import annotations + +import os +from typing import Any, Dict + +import pytest + +from agent.secret_scope import ( + reset_secret_scope, + set_multiplex_active, + set_secret_scope, +) + + +@pytest.fixture +def multiplex_scope(): + """Install a secret scope with multiplexing ON; restore state after.""" + + def _install(scope: Dict[str, str]): + set_multiplex_active(True) + token = set_secret_scope(scope) + return token + + tokens = [] + + def install(scope: Dict[str, str]): + tokens.append(_install(scope)) + + yield install + + for token in tokens: + reset_secret_scope(token) + set_multiplex_active(False) + + +# --------------------------------------------------------------------------- +# Family A — memory plugins +# --------------------------------------------------------------------------- + +class TestMemoryFamily: + def test_retaindb_scoped_key_wins(self, multiplex_scope, monkeypatch): + monkeypatch.setenv("RETAINDB_API_KEY", "env-other-profile") + multiplex_scope({"RETAINDB_API_KEY": "scoped-key"}) + + from plugins.memory.retaindb import RetainDBMemoryProvider + + assert RetainDBMemoryProvider().is_available() is True + + def test_retaindb_scoped_miss_does_not_borrow_environ( + self, multiplex_scope, monkeypatch + ): + # Env holds another profile's key; the active profile's scope has none. + monkeypatch.setenv("RETAINDB_API_KEY", "env-other-profile") + multiplex_scope({}) + + from plugins.memory.retaindb import RetainDBMemoryProvider + + assert RetainDBMemoryProvider().is_available() is False + + def test_supermemory_scoped_miss_does_not_borrow_environ( + self, multiplex_scope, monkeypatch + ): + monkeypatch.setenv("SUPERMEMORY_API_KEY", "env-other-profile") + multiplex_scope({}) + + from plugins.memory.supermemory import SupermemoryMemoryProvider + + assert SupermemoryMemoryProvider().is_available() is False + + def test_supermemory_post_setup_no_environ_write_under_multiplex( + self, multiplex_scope, monkeypatch, tmp_path + ): + """post_setup must not pollute process env with a profile's key.""" + multiplex_scope({}) + monkeypatch.delenv("SUPERMEMORY_API_KEY", raising=False) + + import hermes_cli.config as cli_config + import hermes_cli.memory_setup as memory_setup + import plugins.memory.supermemory as sm + + monkeypatch.setattr(memory_setup, "_prompt", lambda *a, **k: "sm-fresh-key") + monkeypatch.setattr(memory_setup, "_write_env_vars", lambda *a, **k: None) + monkeypatch.setattr(cli_config, "save_config", lambda *a, **k: None) + monkeypatch.setattr( + sm, + "_probe_supermemory_connection", + lambda *a, **k: {"ok": True, "detail": "stub"}, + ) + monkeypatch.setattr(sm, "_format_connection_summary", lambda s: "stub") + + sm.SupermemoryMemoryProvider().post_setup(str(tmp_path), {}) + + assert "SUPERMEMORY_API_KEY" not in os.environ + + def test_supermemory_post_setup_environ_write_kept_single_profile( + self, monkeypatch, tmp_path + ): + """Single-profile (multiplex off): the convenience write still happens.""" + set_multiplex_active(False) + monkeypatch.delenv("SUPERMEMORY_API_KEY", raising=False) + + import hermes_cli.config as cli_config + import hermes_cli.memory_setup as memory_setup + import plugins.memory.supermemory as sm + + monkeypatch.setattr(memory_setup, "_prompt", lambda *a, **k: "sm-fresh-key") + monkeypatch.setattr(memory_setup, "_write_env_vars", lambda *a, **k: None) + monkeypatch.setattr(cli_config, "save_config", lambda *a, **k: None) + monkeypatch.setattr( + sm, + "_probe_supermemory_connection", + lambda *a, **k: {"ok": True, "detail": "stub"}, + ) + monkeypatch.setattr(sm, "_format_connection_summary", lambda s: "stub") + + try: + sm.SupermemoryMemoryProvider().post_setup(str(tmp_path), {}) + assert os.environ.get("SUPERMEMORY_API_KEY") == "sm-fresh-key" + finally: + os.environ.pop("SUPERMEMORY_API_KEY", None) + + +# --------------------------------------------------------------------------- +# Family B — image_gen plugins +# --------------------------------------------------------------------------- + +class TestImageGenFamily: + def test_deepinfra_scoped_key_wins(self, multiplex_scope, monkeypatch): + monkeypatch.delenv("DEEPINFRA_API_KEY", raising=False) + multiplex_scope({"DEEPINFRA_API_KEY": "scoped-key"}) + + from plugins.image_gen.deepinfra import DeepInfraImageGenProvider + + assert DeepInfraImageGenProvider().is_available() is True + + def test_deepinfra_scoped_miss_does_not_borrow_environ( + self, multiplex_scope, monkeypatch + ): + monkeypatch.setenv("DEEPINFRA_API_KEY", "env-other-profile") + multiplex_scope({}) + + from plugins.image_gen.deepinfra import DeepInfraImageGenProvider + + assert DeepInfraImageGenProvider().is_available() is False + + +# --------------------------------------------------------------------------- +# Family C — browser/web plugins +# --------------------------------------------------------------------------- + +class TestBrowserFamily: + def test_firecrawl_scoped_key_wins(self, multiplex_scope, monkeypatch): + monkeypatch.delenv("FIRECRAWL_API_KEY", raising=False) + multiplex_scope({"FIRECRAWL_API_KEY": "scoped-key"}) + + from plugins.browser.firecrawl.provider import FirecrawlBrowserProvider + + provider = FirecrawlBrowserProvider() + assert provider.is_available() is True + assert provider._headers()["Authorization"] == "Bearer scoped-key" + + def test_firecrawl_scoped_miss_does_not_borrow_environ( + self, multiplex_scope, monkeypatch + ): + monkeypatch.setenv("FIRECRAWL_API_KEY", "env-other-profile") + multiplex_scope({}) + + from plugins.browser.firecrawl.provider import FirecrawlBrowserProvider + + assert FirecrawlBrowserProvider().is_available() is False + + +# --------------------------------------------------------------------------- +# Family E — google_meet spawn-wrap +# --------------------------------------------------------------------------- + +class TestGoogleMeetSpawn: + def test_child_env_carries_scoped_openai_key( + self, multiplex_scope, monkeypatch, tmp_path + ): + """start() resolves the key from the scope and injects it explicitly.""" + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.delenv("HERMES_MEET_REALTIME_KEY", raising=False) + multiplex_scope({"OPENAI_API_KEY": "scoped-openai-key"}) + + import plugins.google_meet.process_manager as pm + + monkeypatch.setattr(pm, "_root", lambda: tmp_path) + + captured: Dict[str, Any] = {} + + class _FakeProc: + pid = 4242 + + def fake_popen(cmd, **kwargs): + captured["env"] = kwargs.get("env") + return _FakeProc() + + monkeypatch.setattr(pm.subprocess, "Popen", fake_popen) + + result = pm.start( + "https://meet.google.com/abc-defg-hij", + out_dir=tmp_path / "meeting", + mode="realtime", + ) + + assert result["ok"] is True + child_env = captured["env"] + # The scoped key crosses the process boundary explicitly, not via + # inherited os.environ (which had no key at all). + assert child_env["HERMES_MEET_REALTIME_KEY"] == "scoped-openai-key" + + def test_explicit_key_argument_still_wins( + self, multiplex_scope, monkeypatch, tmp_path + ): + multiplex_scope({"OPENAI_API_KEY": "scoped-openai-key"}) + + import plugins.google_meet.process_manager as pm + + monkeypatch.setattr(pm, "_root", lambda: tmp_path) + + captured: Dict[str, Any] = {} + + class _FakeProc: + pid = 4243 + + monkeypatch.setattr( + pm.subprocess, + "Popen", + lambda cmd, **kw: (captured.update(env=kw.get("env")), _FakeProc())[1], + ) + + pm.start( + "https://meet.google.com/abc-defg-hij", + out_dir=tmp_path / "meeting2", + realtime_api_key="explicit-key", + ) + + assert captured["env"]["HERMES_MEET_REALTIME_KEY"] == "explicit-key" diff --git a/tests/test_tui_gateway_server.py b/tests/test_tui_gateway_server.py index e2cde67eaddc..c5879a042793 100644 --- a/tests/test_tui_gateway_server.py +++ b/tests/test_tui_gateway_server.py @@ -11153,6 +11153,104 @@ def _fake_make_agent(*a, **k): server._sessions.pop(k, None) +def test_session_branch_installs_parent_profile_secret_scope(monkeypatch, tmp_path): + """The branched agent must be built under the parent profile's secrets. + + session.branch already binds the parent's HERMES_HOME and state.db, but the + secret scope is what makes get_secret() resolve that profile's .env. Without + it the build falls through to process os.environ — the LAUNCH profile's + credentials — which is exactly the cross-profile resolution #67605 fixed for + session.create / session.resume. + """ + import threading + + from agent.secret_scope import current_secret_scope + + profile_home = tmp_path / "profiles" / "mlperf" + profile_home.mkdir(parents=True) + (profile_home / ".env").write_text( + "PROXMOX_TOKEN=mlperf-secret\n", encoding="utf-8" + ) + seen: dict = {"msgs": []} + + class ProfileDB: + def __init__(self, db_path=None): + pass + + def get_session_title(self, _key): + return "parent" + + def get_next_title_in_lineage(self, current): + return f"{current} (branch)" + + def create_session(self, new_key, **kwargs): + seen["created"] = new_key + + def append_message(self, **kwargs): + seen["msgs"].append(kwargs) + + def set_session_title(self, key, title): + return True + + def get_session(self, key): + return {"id": key, "cwd": str(tmp_path)} + + def update_session_cwd(self, *a, **k): + return None + + def close(self): + return None + + class FakeAgent: + def __init__(self): + self.model = "test-model" + self.session_id = None + + parent = { + "session_key": "parent-key", + "history": [{"role": "user", "content": "hi"}], + "history_lock": threading.Lock(), + "running": False, + "cols": 80, + "profile_home": str(profile_home), + "source": "tui", + "agent": FakeAgent(), + "created_at": 1.0, + "last_active": 1.0, + "cwd": str(tmp_path), + } + server._sessions["parent"] = parent + monkeypatch.setattr(server, "_get_db", lambda: ProfileDB()) + monkeypatch.setattr("hermes_state.SessionDB", ProfileDB) + monkeypatch.setattr(server, "_claim_active_session_slot", lambda *a, **k: (None, None)) + + def _fake_make_agent(*a, **k): + scope = current_secret_scope() + seen["scope"] = dict(scope) if scope else None + return FakeAgent() + + monkeypatch.setattr(server, "_make_agent", _fake_make_agent) + monkeypatch.setattr(server, "_set_session_context", lambda *a, **k: {}) + monkeypatch.setattr(server, "_clear_session_context", lambda *a, **k: None) + monkeypatch.setattr(server, "_resolve_model", lambda: "test-model") + monkeypatch.setattr(server, "_session_cwd", lambda s: str(tmp_path)) + monkeypatch.setattr(server, "_register_session_cwd", lambda *a, **k: None) + monkeypatch.setattr(server, "_attach_worker", lambda *a, **k: None) + try: + resp = server.handle_request( + { + "id": "1", + "method": "session.branch", + "params": {"session_id": "parent", "name": "forked"}, + } + ) + assert "result" in resp, resp + assert seen.get("scope") == {"PROXMOX_TOKEN": "mlperf-secret"} + finally: + for k in list(server._sessions): + server._sessions.pop(k, None) + + def test_pending_title_finalizer_uses_session_profile_db(monkeypatch, tmp_path): """Post-turn pending_title must land in the session profile store.""" profile_home = tmp_path / "profiles" / "mlperf" @@ -14052,6 +14150,49 @@ def test_reap_idle_sessions_closes_only_evictable(monkeypatch): server._sessions.clear() +def test_reap_idle_sessions_calls_periodic_trim(monkeypatch): + """The idle reaper must call trim_memory every scan, even with no victims.""" + trim_calls = [] + monkeypatch.setattr(server, "_session_pending_kind", lambda sid: "") + monkeypatch.setattr(server, "_close_session_by_id", lambda *a, **k: None) + monkeypatch.setattr(server, "_enforce_session_cap", lambda: None) + monkeypatch.setattr(server, "_reclaim_orphaned_leases", lambda: None) + + # Patch the delayed import path: the function does + # `from hermes_cli.mem_trim import trim_memory` at call time. + import hermes_cli.mem_trim as mem_trim + + monkeypatch.setattr( + mem_trim, "trim_memory", + lambda **kw: trim_calls.append(kw.get("reason", "")) or True, + ) + + server._sessions.clear() + try: + server._reap_idle_sessions() + assert len(trim_calls) == 1 + assert trim_calls[0] == "idle reaper periodic trim" + finally: + server._sessions.clear() + + +def test_reap_idle_sessions_logs_trim_failure(monkeypatch, caplog): + monkeypatch.setattr(server, "_session_pending_kind", lambda sid: "") + monkeypatch.setattr(server, "_close_session_by_id", lambda *a, **k: None) + monkeypatch.setattr(server, "_enforce_session_cap", lambda: None) + monkeypatch.setattr(server, "_reclaim_orphaned_leases", lambda: None) + import hermes_cli.mem_trim as mem_trim + + monkeypatch.setattr(mem_trim, "trim_memory", lambda **_kw: (_ for _ in ()).throw(RuntimeError("boom"))) + server._sessions.clear() + try: + with caplog.at_level("DEBUG", logger="tui_gateway.server"): + server._reap_idle_sessions() + assert "idle reaper memory trim failed: RuntimeError: boom" in caplog.text + finally: + server._sessions.clear() + + def test_session_create_records_ui_model_as_session_override(monkeypatch): """The desktop composer owns its model as plain UI state and ships it on session.create. The gateway must record it as a PER-SESSION override (built @@ -15438,3 +15579,76 @@ def start(self): assert captured.get("persist_user_message") == "hi" finally: server._sessions.pop("sid", None) + + +def test_prompt_submit_releases_old_history_before_heap_trim(monkeypatch): + """The trim boundary must not retain the just-pruned history snapshots.""" + observed = {} + cleanup_order = [] + + class _Agent: + def run_conversation( + self, prompt, conversation_history=None, stream_callback=None + ): + return { + "final_response": "reply", + "messages": [{"role": "assistant", "content": "reply"}], + } + + class _ImmediateThread: + def __init__(self, target=None, daemon=None): + self._target = target + + def start(self): + assert self._target is not None + self._target() + + def _inspect_trim_frame(**_kwargs): + import inspect + + cleanup_order.append("trim") + frame = inspect.currentframe() + assert frame is not None and frame.f_back is not None + caller_locals = frame.f_back.f_locals + # Loud, not vacuous: if the production locals are ever renamed, fail + # the test instead of silently reading None and "passing". + assert "history" in caller_locals and "run_kwargs" in caller_locals, ( + "expected locals not found in _run_prompt_submit's finally frame — " + "renamed? update this test" + ) + observed["history"] = caller_locals.get("history") + observed["run_kwargs"] = caller_locals.get("run_kwargs") + + session = _session(agent=_Agent()) + session["profile_home"] = "/tmp/test-profile" + session["history"] = [ + {"role": "tool", "tool_call_id": "old", "content": "x" * 20_000} + ] + server._sessions["sid_trim"] = session + try: + monkeypatch.setattr(server.threading, "Thread", _ImmediateThread) + monkeypatch.setattr(server, "_get_usage", lambda _a: {}) + monkeypatch.setattr(server, "render_message", lambda _t, _c: "") + monkeypatch.setattr(server, "_emit", lambda *a: None) + monkeypatch.setattr(server, "set_hermes_home_override", lambda _home: object()) + monkeypatch.setattr( + server, + "reset_hermes_home_override", + lambda _token: cleanup_order.append("reset_home"), + ) + monkeypatch.setattr("hermes_cli.mem_trim.trim_memory", _inspect_trim_frame) + + resp = server.handle_request( + { + "id": "1", + "method": "prompt.submit", + "params": {"session_id": "sid_trim", "text": "hi"}, + } + ) + + assert resp is not None and resp.get("result") + assert not observed["history"] + assert not observed["run_kwargs"] + assert cleanup_order == ["trim", "reset_home"] + finally: + server._sessions.pop("sid_trim", None) diff --git a/tests/test_web_server_status_topology_cache.py b/tests/test_web_server_status_topology_cache.py new file mode 100644 index 000000000000..76c4d0be2ed0 --- /dev/null +++ b/tests/test_web_server_status_topology_cache.py @@ -0,0 +1,117 @@ +"""Regression tests for the /api/status profile-topology cache. + +The desktop app polls /api/status ~1/s while waiting for the backend to become +ready. Before the cache, every poll ran a full _collect_profile_gateway_topology +scan (per-profile yaml.safe_load with the pure-Python loader + psutil +process-table probes + realpath walks) in the default executor; on multi-profile +installs the concurrent scans held the GIL for 14-16s and starved the event +loop, so the desktop WS never received gateway.ready and boot escalated to the +"Hermes couldn't start" overlay (#60800). +""" + +import threading +import time + +from hermes_cli import web_server + + +def _reset_cache(): + web_server._TOPOLOGY_CACHE["ts"] = 0.0 + web_server._TOPOLOGY_CACHE["data"] = None + web_server._TOPOLOGY_CACHE["fn"] = None + + +def _fake_topology(calls, delay=0.0): + def _collect(): + if delay: + time.sleep(delay) + calls.append(1) + return {"profiles": ["default"], "gateway_mode": "single", "gateways": []} + + return _collect + + +def test_topology_cache_returns_cached_result_within_ttl(monkeypatch): + calls = [] + monkeypatch.setattr( + web_server, "_collect_profile_gateway_topology", _fake_topology(calls) + ) + _reset_cache() + try: + first = web_server._collect_profile_gateway_topology_cached() + second = web_server._collect_profile_gateway_topology_cached() + finally: + _reset_cache() + + assert len(calls) == 1 + assert first is second + + +def test_topology_cache_rescans_after_ttl(monkeypatch): + calls = [] + monkeypatch.setattr( + web_server, "_collect_profile_gateway_topology", _fake_topology(calls) + ) + _reset_cache() + try: + web_server._collect_profile_gateway_topology_cached() + # Age the cache entry past the TTL instead of sleeping through it. + web_server._TOPOLOGY_CACHE["ts"] -= web_server._TOPOLOGY_CACHE_TTL + 1.0 + web_server._collect_profile_gateway_topology_cached() + finally: + _reset_cache() + + assert len(calls) == 2 + + +def test_topology_cache_collapses_concurrent_scans(monkeypatch): + """Concurrent status polls must not each run their own scan — that pile-up + is exactly the GIL storm the cache exists to prevent.""" + calls = [] + monkeypatch.setattr( + web_server, + "_collect_profile_gateway_topology", + _fake_topology(calls, delay=0.05), + ) + _reset_cache() + results = [] + try: + threads = [ + threading.Thread( + target=lambda: results.append( + web_server._collect_profile_gateway_topology_cached() + ) + ) + for _ in range(8) + ] + for t in threads: + t.start() + for t in threads: + t.join() + finally: + _reset_cache() + + assert len(calls) == 1 + assert len(results) == 8 + assert all(r == results[0] for r in results) +def test_topology_cache_misses_when_collector_is_swapped(monkeypatch): + """Tests (and hot-reload scenarios) monkeypatch the collector; a swapped + function identity must be a cache miss so stale data from the previous + collector never leaks across the swap.""" + calls_a, calls_b = [], [] + monkeypatch.setattr( + web_server, "_collect_profile_gateway_topology", _fake_topology(calls_a) + ) + _reset_cache() + try: + first = web_server._collect_profile_gateway_topology_cached() + monkeypatch.setattr( + web_server, "_collect_profile_gateway_topology", _fake_topology(calls_b) + ) + second = web_server._collect_profile_gateway_topology_cached() + finally: + _reset_cache() + + assert len(calls_a) == 1 + assert len(calls_b) == 1 + assert first is not second diff --git a/tests/tools/test_file_tools.py b/tests/tools/test_file_tools.py index 329247ec313d..1efe65669808 100644 --- a/tests/tools/test_file_tools.py +++ b/tests/tools/test_file_tools.py @@ -716,3 +716,281 @@ def test_invalidate_evicts_the_task_resolved_key(self, tmp_path, monkeypatch): assert correct not in remaining, remaining ft._read_tracker.pop(task_id, None) + + +# --------------------------------------------------------------------------- +# Negative-result cache tests +# +# Without this cache, a typo'd path retried 13 times (observed in the wild) +# spawned 13 wc -c subprocesses + 13 ls walks for the "did you mean..." hint. +# The cache returns the same error JSON immediately and skips both shells. +# --------------------------------------------------------------------------- + +class TestNotFoundCache: + @patch("tools.file_tools._get_file_ops") + def test_read_caches_file_not_found_and_skips_subprocess_on_retry(self, mock_get): + mock_ops = MagicMock() + result_obj = MagicMock() + result_obj.content = None + # Shape returned by ShellFileOperations._suggest_similar_files + result_obj.to_dict.return_value = { + "error": "File not found: /tmp/does-not-exist-neg-1.txt", + "similar_files": [], + } + mock_ops.read_file.return_value = result_obj + mock_get.return_value = mock_ops + + from tools.file_tools import read_file_tool, _read_tracker + # Use a unique task_id so we don't collide with other tests. + tid = "neg-cache-read-1" + _read_tracker.pop(tid, None) + + # First call: subprocess runs, error returned, cache populated. + first = json.loads(read_file_tool("/tmp/does-not-exist-neg-1.txt", task_id=tid)) + assert "File not found" in first["error"] + assert mock_ops.read_file.call_count == 1 + + # Second call: same path → cache hit → no new subprocess call. + second = json.loads(read_file_tool("/tmp/does-not-exist-neg-1.txt", task_id=tid)) + assert "File not found" in second["error"] + assert mock_ops.read_file.call_count == 1, ( + "Negative cache hit must skip the subprocess on retry" + ) + + @patch("tools.file_tools._get_file_ops") + def test_read_cache_isolated_per_task(self, mock_get): + mock_ops = MagicMock() + result_obj = MagicMock() + result_obj.to_dict.return_value = { + "error": "File not found: /tmp/does-not-exist-neg-2.txt", + "similar_files": [], + } + mock_ops.read_file.return_value = result_obj + mock_get.return_value = mock_ops + + from tools.file_tools import read_file_tool, _read_tracker + for tid in ("neg-cache-iso-A", "neg-cache-iso-B"): + _read_tracker.pop(tid, None) + + read_file_tool("/tmp/does-not-exist-neg-2.txt", task_id="neg-cache-iso-A") + read_file_tool("/tmp/does-not-exist-neg-2.txt", task_id="neg-cache-iso-B") + # Each task gets its own miss; B doesn't reuse A's cache entry. + assert mock_ops.read_file.call_count == 2 + + @patch("tools.file_tools._get_file_ops") + def test_read_cache_populated_only_for_not_found(self, mock_get): + # A successful read must NOT populate the negative cache. + mock_ops = MagicMock() + result_obj = MagicMock() + result_obj.content = "x" + result_obj.to_dict.return_value = {"content": "x", "total_lines": 1} + mock_ops.read_file.return_value = result_obj + mock_get.return_value = mock_ops + + from tools.file_tools import read_file_tool, _read_tracker + tid = "neg-cache-success-only" + _read_tracker.pop(tid, None) + + read_file_tool("/tmp/exists-or-mocked.txt", task_id=tid) + nf = _read_tracker[tid].get("not_found", {}) + assert all(k[0] != "read" or "exists-or-mocked" not in k[1] for k in nf), ( + "Successful reads must not poison the negative cache" + ) + + @patch("tools.file_tools._get_file_ops") + def test_search_caches_path_not_found_and_skips_subprocess_on_retry(self, mock_get): + mock_ops = MagicMock() + result_obj = MagicMock() + result_obj.matches = [] + result_obj.to_dict.return_value = { + "error": "Path not found: /tmp/does-not-exist-search-3", + "total_count": 0, + } + mock_ops.search.return_value = result_obj + mock_get.return_value = mock_ops + + from tools.file_tools import search_tool, _read_tracker + tid = "neg-cache-search-3" + _read_tracker.pop(tid, None) + + first = json.loads(search_tool("foo", path="/tmp/does-not-exist-search-3", task_id=tid)) + assert "Path not found" in first["error"] + assert mock_ops.search.call_count == 1 + + second = json.loads(search_tool("foo", path="/tmp/does-not-exist-search-3", task_id=tid)) + assert "Path not found" in second["error"] + assert mock_ops.search.call_count == 1, ( + "Search negative cache hit must skip the subprocess on retry" + ) + + @patch("tools.file_tools._get_file_ops") + def test_read_and_search_caches_are_namespaced(self, mock_get): + # A read that misses must NOT serve a subsequent search call's miss + # (different error JSON shapes). + mock_ops = MagicMock() + + read_obj = MagicMock() + read_obj.to_dict.return_value = { + "error": "File not found: /tmp/does-not-exist-namespace-4", + } + mock_ops.read_file.return_value = read_obj + + search_obj = MagicMock() + search_obj.matches = [] + search_obj.to_dict.return_value = { + "error": "Path not found: /tmp/does-not-exist-namespace-4", + "total_count": 0, + } + mock_ops.search.return_value = search_obj + + mock_get.return_value = mock_ops + + from tools.file_tools import read_file_tool, search_tool, _read_tracker + tid = "neg-cache-namespace-4" + _read_tracker.pop(tid, None) + + read_file_tool("/tmp/does-not-exist-namespace-4", task_id=tid) + search_tool("foo", path="/tmp/does-not-exist-namespace-4", task_id=tid) + # Both ops must hit their own caller (namespacing prevents read's + # error JSON from being returned to search). + assert mock_ops.read_file.call_count == 1 + assert mock_ops.search.call_count == 1 + + @patch("tools.file_tools._get_file_ops") + def test_write_invalidates_read_negative_cache(self, mock_get): + # After write_file on a path, a subsequent read must hit disk, + # not return the cached "not found" stub. + mock_ops = MagicMock() + + not_found_obj = MagicMock() + not_found_obj.to_dict.return_value = { + "error": "File not found: /tmp/will-be-created-neg-5.txt", + } + present_obj = MagicMock() + present_obj.content = "after write" + present_obj.to_dict.return_value = {"content": "after write", "total_lines": 1} + + # First read → not found; second read (after write) → present. + mock_ops.read_file.side_effect = [not_found_obj, present_obj] + write_result_obj = MagicMock() + write_result_obj.to_dict.return_value = {"status": "ok"} + mock_ops.write_file.return_value = write_result_obj + mock_get.return_value = mock_ops + + from tools.file_tools import read_file_tool, write_file_tool, _read_tracker + tid = "neg-cache-write-invalidate-5" + _read_tracker.pop(tid, None) + + first = json.loads(read_file_tool("/tmp/will-be-created-neg-5.txt", task_id=tid)) + assert "File not found" in first["error"] + + write_file_tool("/tmp/will-be-created-neg-5.txt", "after write", task_id=tid) + + second = json.loads(read_file_tool("/tmp/will-be-created-neg-5.txt", task_id=tid)) + assert second.get("content") == "after write", ( + "write_file must invalidate the negative cache so the next read " + "hits the now-existing file instead of returning a stale stub" + ) + assert mock_ops.read_file.call_count == 2 + + def test_not_found_ttl_expires(self): + # A cache entry older than _NOT_FOUND_TTL_SECONDS must be discarded. + from tools.file_tools import ( + _check_not_found_cache, + _record_not_found, + _read_tracker, + _NOT_FOUND_TTL_SECONDS, + ) + import tools.file_tools as ft + + tid = "neg-cache-ttl-6" + _read_tracker.pop(tid, None) + _record_not_found("read", "/tmp/ttl-test", tid, '{"error":"x"}') + # Fresh entry: cache hit. + assert _check_not_found_cache("read", "/tmp/ttl-test", tid) is not None + + # Backdate the entry past the TTL. + with ft._read_tracker_lock: + entry = _read_tracker[tid]["not_found"][("read", "/tmp/ttl-test")] + ft._read_tracker[tid]["not_found"][("read", "/tmp/ttl-test")] = ( + entry[0] - _NOT_FOUND_TTL_SECONDS - 1.0, + entry[1], + ) + # Stale entry: cache miss, also evicted. + assert _check_not_found_cache("read", "/tmp/ttl-test", tid) is None + with ft._read_tracker_lock: + assert ("read", "/tmp/ttl-test") not in _read_tracker[tid].get("not_found", {}) + + def test_out_of_band_creation_defeats_cached_miss(self, tmp_path): + """CRITICAL staleness contract: a file created AFTER a cached miss — + by a terminal command or any external process, NOT write_file_tool — + must be served for real on the next read. The agent pattern + 'check for file → create it → read it' breaks otherwise.""" + from tools.file_tools import ( + _check_not_found_cache, + _record_not_found, + _read_tracker, + ) + + tid = "neg-cache-oob-read" + _read_tracker.pop(tid, None) + target = tmp_path / "created-later.txt" + + _record_not_found("read", str(target), tid, '{"error":"File not found: x"}') + assert _check_not_found_cache("read", str(target), tid) is not None + + # Out-of-band creation: plain filesystem write, no tool hook fires. + target.write_text("real content\n") + + # The cached miss must NOT be served once the path exists… + assert _check_not_found_cache("read", str(target), tid) is None, ( + "stale 'File not found' served after the file was created " + "out-of-band — the existence guard regressed" + ) + # …and the entry is evicted, not just skipped. + with __import__("tools.file_tools", fromlist=["x"])._read_tracker_lock: + assert ("read", str(target)) not in _read_tracker[tid].get("not_found", {}) + + def test_out_of_band_creation_defeats_cached_search_miss(self, tmp_path): + """Same contract for search roots: creating a file under a + previously-missing directory must defeat the cached 'Path not found'.""" + from tools.file_tools import ( + _check_not_found_cache, + _record_not_found, + _read_tracker, + ) + + tid = "neg-cache-oob-search" + _read_tracker.pop(tid, None) + missing_dir = tmp_path / "later-dir" + + _record_not_found("search", str(missing_dir), tid, '{"error":"Path not found: x"}') + assert _check_not_found_cache("search", str(missing_dir), tid) is not None + + missing_dir.mkdir() + (missing_dir / "x.txt").write_text("hi\n") + + assert _check_not_found_cache("search", str(missing_dir), tid) is None, ( + "stale 'Path not found' served after the directory was created" + ) + + def test_notify_other_tool_call_clears_not_found(self): + """Belt-and-suspenders: any non-read tool (terminal etc.) invalidates + the task's negative cache via the dispatcher's notify hook.""" + from tools.file_tools import ( + _check_not_found_cache, + _record_not_found, + _read_tracker, + notify_other_tool_call, + ) + + tid = "neg-cache-notify" + _read_tracker.pop(tid, None) + _record_not_found("read", "/tmp/never-exists-notify", tid, '{"error":"x"}') + assert _check_not_found_cache("read", "/tmp/never-exists-notify", tid) is not None + + notify_other_tool_call(tid) + + assert _check_not_found_cache("read", "/tmp/never-exists-notify", tid) is None, ( + "notify_other_tool_call must clear cached misses" + ) diff --git a/tests/tools/test_stage2_hook_symlink_chown.py b/tests/tools/test_stage2_hook_symlink_chown.py index cc8d0312d9ac..fd68d78def7f 100644 --- a/tests/tools/test_stage2_hook_symlink_chown.py +++ b/tests/tools/test_stage2_hook_symlink_chown.py @@ -114,3 +114,15 @@ def test_stage2_skips_top_level_chown_for_symlinked_hermes_home( stage2_text: str, ) -> None: assert 'refuse_symlinked_path "chown" "$HERMES_HOME"' in stage2_text + + +def test_stage2_skips_recursive_repairs_when_tree_is_already_owned( + stage2_text: str, +) -> None: + assert "tree_has_non_hermes_owner() {" in stage2_text + assert 'if [ -e "$HERMES_HOME/$sub" ] && tree_has_non_hermes_owner "$HERMES_HOME/$sub"; then' in stage2_text + assert 'if [ -d "$HERMES_HOME/profiles" ] && tree_has_non_hermes_owner "$HERMES_HOME/profiles"; then' in stage2_text + # Sibling every-boot chown blocks carry the same warm-boot gate. + assert 'if [ -d "$HERMES_HOME/cron" ] && tree_has_non_hermes_owner "$HERMES_HOME/cron"; then' in stage2_text + assert 'if [ -d "$HERMES_HOME/platforms/pairing" ] && tree_has_non_hermes_owner "$HERMES_HOME/platforms/pairing"; then' in stage2_text + assert 'if [ -d "$HERMES_HOME/pairing" ] && tree_has_non_hermes_owner "$HERMES_HOME/pairing"; then' in stage2_text diff --git a/tests/tools/test_voice_mode.py b/tests/tools/test_voice_mode.py index 51d921d61342..2d693a7f6ef5 100644 --- a/tests/tools/test_voice_mode.py +++ b/tests/tools/test_voice_mode.py @@ -207,6 +207,11 @@ def test_wsl_without_pulse_blocks_voice(self, monkeypatch, tmp_path): monkeypatch.setattr("tools.voice_mode._pulse_socket_reachable", lambda: False) monkeypatch.setattr("tools.voice_mode._import_audio", lambda: (MagicMock(), MagicMock())) + # Pin the WSL2 PowerShell TTS fallback: without this the no-forwarding + # block depends on whether powershell.exe + ffmpeg exist on the host + # (a WSL machine with both sees available=True via the fallback and + # this test flakes). The fallback path is covered by its own test. + monkeypatch.setattr("tools.voice_mode._wsl_powershell_tts_available", lambda: False) proc_version = tmp_path / "proc_version" proc_version.write_text("Linux 5.15.0-microsoft-standard-WSL2") diff --git a/tests/tools/test_voice_wsl_pipewire.py b/tests/tools/test_voice_wsl_pipewire.py index 395d401fbf4e..ff82e7f208c3 100644 --- a/tests/tools/test_voice_wsl_pipewire.py +++ b/tests/tools/test_voice_wsl_pipewire.py @@ -28,6 +28,11 @@ def _base(monkeypatch): monkeypatch.delenv("PIPEWIRE_REMOTE", raising=False) monkeypatch.setattr("hermes_constants.is_container", lambda: False) monkeypatch.setattr("tools.voice_mode._pulse_socket_reachable", lambda: False) + # The WSL2 PowerShell TTS fallback makes the no-forwarding path host- + # dependent (powershell.exe + ffmpeg on PATH). Pin it so these tests are + # deterministic on any machine: the fallback is only exercised by the + # dedicated test that mocks it to True. + monkeypatch.setattr("tools.voice_mode._wsl_powershell_tts_available", lambda: False) sd = MagicMock(); sd.query_devices.return_value = [{"name": "dev"}] monkeypatch.setattr("tools.voice_mode._import_audio", lambda: (sd, MagicMock())) @@ -56,3 +61,23 @@ def test_wsl_without_forwarding_still_blocks(monkeypatch): res = detect_audio_environment() assert res["available"] is False assert any("WSL" in w for w in res["warnings"]) + + +def test_wsl_without_forwarding_but_powershell_fallback_allows_tts(monkeypatch): + """The WSL2 PowerShell TTS fallback relaxes the no-forwarding block for + OUTPUT (TTS playback) while keeping recording guidance visible. + + Regression for the stale-test bug: this path was added by the PowerShell + fallback feature but never unit-tested, so a WSL host with powershell.exe + and ffmpeg (the fallback's only preconditions) flipped the no-forwarding + test from pass to fail depending on the machine. + """ + _base(monkeypatch) + _force_wsl(monkeypatch) # no PULSE_SERVER, no PIPEWIRE_REMOTE + from tools.voice_mode import detect_audio_environment + monkeypatch.setattr("tools.voice_mode._wsl_powershell_tts_available", lambda: True) + res = detect_audio_environment() + # TTS playback works via Media.SoundPlayer on the Windows host, so voice + # mode is available — but the WSL guidance notice must still surface. + assert res["available"] is True + assert any("WSL" in w for w in res["notices"]) diff --git a/tests/tools/test_xai_http_credentials.py b/tests/tools/test_xai_http_credentials.py new file mode 100644 index 000000000000..4217823f8c83 --- /dev/null +++ b/tests/tools/test_xai_http_credentials.py @@ -0,0 +1,57 @@ +import pytest + + +def _set_xai_oauth_unavailable(monkeypatch): + from hermes_cli import auth + + monkeypatch.setattr(auth, "resolve_xai_oauth_runtime_credentials", lambda **_: {}) + + +def test_xai_credentials_fail_closed_without_profile_scope(tmp_path, monkeypatch): + from agent import secret_scope + from hermes_cli.config import invalidate_env_cache + from tools.xai_http import resolve_xai_http_credentials + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("XAI_API_KEY", "foreign-xai-key") + monkeypatch.setenv("XAI_BASE_URL", "https://foreign.example/v1") + _set_xai_oauth_unavailable(monkeypatch) + invalidate_env_cache() + previous_multiplex = secret_scope.is_multiplex_active() + token = secret_scope.set_secret_scope(None) + secret_scope.set_multiplex_active(True) + try: + with pytest.raises(secret_scope.UnscopedSecretError): + resolve_xai_http_credentials(force_refresh=True) + finally: + secret_scope.reset_secret_scope(token) + secret_scope.set_multiplex_active(previous_multiplex) + invalidate_env_cache() + + +def test_xai_credentials_do_not_fall_back_to_environ_when_scope_has_no_key( + tmp_path, monkeypatch +): + from agent import secret_scope + from hermes_cli.config import invalidate_env_cache + from tools.xai_http import resolve_xai_http_credentials + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("XAI_API_KEY", "foreign-xai-key") + monkeypatch.setenv("XAI_BASE_URL", "https://foreign.example/v1") + _set_xai_oauth_unavailable(monkeypatch) + invalidate_env_cache() + previous_multiplex = secret_scope.is_multiplex_active() + token = secret_scope.set_secret_scope({}) + secret_scope.set_multiplex_active(True) + try: + credentials = resolve_xai_http_credentials(force_refresh=True) + assert credentials == { + "provider": "xai", + "api_key": "", + "base_url": "https://api.x.ai/v1", + } + finally: + secret_scope.reset_secret_scope(token) + secret_scope.set_multiplex_active(previous_multiplex) + invalidate_env_cache() diff --git a/tests/tui_gateway/test_compute_host.py b/tests/tui_gateway/test_compute_host.py index eb748a593611..fa0019722f13 100644 --- a/tests/tui_gateway/test_compute_host.py +++ b/tests/tui_gateway/test_compute_host.py @@ -6,6 +6,10 @@ import threading from pathlib import Path +import pytest + +from tui_gateway.compute_host import ComputeHost, HostSession + def _stdout_queue(proc: subprocess.Popen) -> queue.Queue[dict]: out: queue.Queue[dict] = queue.Queue() @@ -83,3 +87,42 @@ def test_compute_host_line_json_seed_turn_interrupt(): finally: if proc.poll() is None: proc.kill() + + +@pytest.mark.parametrize("kind", ["legacy", "hard-only", "dynamic-getattr"]) +def test_compute_host_interrupt_uses_explicit_stop_compatibility(kind): + calls = [] + + class _Legacy: + def interrupt(self): + calls.append("legacy") + + class _HardOnly: + def hard_interrupt(self): + calls.append("hard") + + class _Dynamic: + def interrupt(self): + calls.append("legacy") + + def __getattr__(self, name): + if name == "hard_interrupt": + return lambda: calls.append("fabricated-hard") + raise AttributeError(name) + + agent = { + "legacy": _Legacy(), + "hard-only": _HardOnly(), + "dynamic-getattr": _Dynamic(), + }[kind] + host = ComputeHost(heartbeat_secs=0) + host._sessions["s1"] = HostSession(sid="s1", agent=agent) + emitted = [] + host.emit = emitted.append + try: + host._handle_interrupt({"sid": "s1", "request_id": "stop"}) + finally: + host.close() + + assert calls == ["hard" if kind == "hard-only" else "legacy"] + assert emitted[-1]["applied"] is True diff --git a/tests/tui_gateway/test_protocol.py b/tests/tui_gateway/test_protocol.py index 23f285523102..00678f4b79b3 100644 --- a/tests/tui_gateway/test_protocol.py +++ b/tests/tui_gateway/test_protocol.py @@ -118,7 +118,54 @@ def test_err_envelope(server): } -# ── write_json ─────────────────────────────────────────────────────── +@pytest.mark.parametrize("kind", ["legacy", "hard-only", "dynamic-getattr"]) +def test_session_interrupt_uses_explicit_stop_compatibility(server, monkeypatch, kind): + calls = [] + + class _Legacy: + def interrupt(self): + calls.append("legacy") + + class _HardOnly: + def hard_interrupt(self): + calls.append("hard") + + class _Dynamic: + def interrupt(self): + calls.append("legacy") + + def __getattr__(self, name): + if name == "hard_interrupt": + return lambda: calls.append("fabricated-hard") + raise AttributeError(name) + + agent = { + "legacy": _Legacy(), + "hard-only": _HardOnly(), + "dynamic-getattr": _Dynamic(), + }[kind] + session = { + "agent": agent, + "history_lock": threading.Lock(), + "running": True, + "queued_prompt": "later", + "session_key": "session-key", + "_run_thread": None, + } + monkeypatch.setattr(server, "_tts_stream_stop", lambda: None) + monkeypatch.setattr(server, "_sess_nowait", lambda _params, _rid: (session, None)) + monkeypatch.setattr(server, "_sess", lambda _params, _rid: (session, None)) + monkeypatch.setattr(server, "_session_uses_compute_host", lambda _session: False) + monkeypatch.setattr(server, "_clear_pending", lambda _sid: None) + response = server._methods["session.interrupt"]( + "stop", {"session_id": "ui-session"} + ) + + assert response["result"]["status"] == "interrupted" + assert calls == ["hard" if kind == "hard-only" else "legacy"] + + +# ── write_json ──────────────────────────────────────────────── def test_write_json(capture): @@ -552,13 +599,13 @@ def test_skin_live_switch_end_to_end(server, tmp_path, monkeypatch): monkeypatch.setattr(server, "_emit", lambda ev, sid, payload=None: emitted.append((ev, payload))) # Baseline (default) — seeds the signature. - (tmp_path / "config.yaml").write_text("display:\n skin: default\n") + (tmp_path / "config.yaml").write_text("display:\n skin: default\n", encoding="utf-8") server._broadcast_skin_if_changed() emitted.clear() # Activate midnight, as `hermes config set display.skin midnight` would. time.sleep(0.01) # ensure the config mtime moves - (tmp_path / "config.yaml").write_text("display:\n skin: midnight\n") + (tmp_path / "config.yaml").write_text("display:\n skin: midnight\n", encoding="utf-8") server._broadcast_skin_if_changed() assert [ev for ev, _ in emitted] == ["skin.changed"] diff --git a/tools/delegate_tool.py b/tools/delegate_tool.py index e75727e602c8..18e47d429db1 100644 --- a/tools/delegate_tool.py +++ b/tools/delegate_tool.py @@ -33,6 +33,7 @@ from urllib.parse import urlsplit, urlunsplit from toolsets import TOOLSETS +from agent.interrupt_compat import request_hard_interrupt # Sentinel value used by the runtime provider system for providers that are # not natively known (named custom providers, third-party aggregators, etc.). @@ -196,7 +197,8 @@ def interrupt_subagent(subagent_id: str) -> bool: if agent is None: return False try: - agent.interrupt(f"Interrupted via TUI ({subagent_id})") + if not request_hard_interrupt(agent, f"Interrupted via TUI ({subagent_id})"): + return False except Exception as exc: logger.debug("interrupt_subagent(%s) failed: %s", subagent_id, exc) return False @@ -2193,9 +2195,8 @@ def _run_with_thread_capture(): except Exception as _timeout_exc: # Signal the child to stop so its thread can exit cleanly. try: - if hasattr(child, "interrupt"): - child.interrupt() - elif hasattr(child, "_interrupt_requested"): + interrupted = child is not None and request_hard_interrupt(child) + if not interrupted and child is not None and hasattr(child, "_interrupt_requested"): child._interrupt_requested = True except Exception: pass @@ -3275,9 +3276,8 @@ def _batch_runner(): def _batch_interrupt(): for _c in _child_agents: try: - if hasattr(_c, "interrupt"): - _c.interrupt("Async delegation cancelled") - elif hasattr(_c, "_interrupt_requested"): + interrupted = request_hard_interrupt(_c, "Async delegation cancelled") + if not interrupted and hasattr(_c, "_interrupt_requested"): _c._interrupt_requested = True except Exception: pass diff --git a/tools/environments/docker.py b/tools/environments/docker.py index 137d80627c3a..780d571b71f8 100644 --- a/tools/environments/docker.py +++ b/tools/environments/docker.py @@ -886,10 +886,10 @@ def __init__( self._container_name: str = "" self._image_uses_s6_init: bool = False self._all_run_args: list[str] = [] - logger.info(f"DockerEnvironment volumes: {volumes}") + logger.info("DockerEnvironment volumes: %s", volumes) # Ensure volumes is a list (config.yaml could be malformed) if volumes is not None and not isinstance(volumes, list): - logger.warning(f"docker_volumes config is not a list: {volumes!r}") + logger.warning("docker_volumes config is not a list: %r", volumes) volumes = [] # Fail fast if Docker is not available. @@ -933,7 +933,7 @@ def __init__( workspace_explicitly_mounted = False for vol in (volumes or []): if not isinstance(vol, str): - logger.warning(f"Docker volume entry is not a string: {vol!r}") + logger.warning("Docker volume entry is not a string: %r", vol) continue vol = vol.strip() if not vol: @@ -943,7 +943,7 @@ def __init__( if ":/workspace" in vol: workspace_explicitly_mounted = True else: - logger.warning(f"Docker volume '{vol}' missing colon, skipping") + logger.warning("Docker volume '%s' missing colon, skipping", vol) host_cwd_abs = os.path.abspath(os.path.expanduser(host_cwd)) if host_cwd else "" bind_host_cwd = ( @@ -953,7 +953,7 @@ def __init__( and not workspace_explicitly_mounted ) if auto_mount_cwd and host_cwd and not os.path.isdir(host_cwd_abs): - logger.debug(f"Skipping docker cwd mount: host_cwd is not a valid directory: {host_cwd}") + logger.debug("Skipping docker cwd mount: host_cwd is not a valid directory: %s", host_cwd) self._workspace_dir: Optional[str] = None self._home_dir: Optional[str] = None @@ -982,7 +982,7 @@ def __init__( ]) if bind_host_cwd: - logger.info(f"Mounting configured host cwd to /workspace: {host_cwd_abs}") + logger.info("Mounting configured host cwd to /workspace: %s", host_cwd_abs) volume_args = ["-v", f"{host_cwd_abs}:/workspace", *volume_args] elif workspace_explicitly_mounted: logger.debug("Skipping docker cwd mount: /workspace already mounted by user config") @@ -1298,7 +1298,7 @@ def __init__( run_exec=image_uses_s6_init, ) - logger.info(f"Docker volume_args: {volume_args}") + logger.info("Docker volume_args: %s", volume_args) # User-supplied extra docker run flags (docker_extra_args in config.yaml). # Appended last so they can override defaults if needed. validated_extra = [] @@ -1336,7 +1336,7 @@ def __init__( + env_args + validated_extra ) - logger.info(f"Docker run_args: {all_run_args}") + logger.info("Docker run_args: %s", all_run_args) # Start the container directly via `docker run -d`. container_name = f"hermes-{uuid.uuid4().hex[:8]}" @@ -1465,7 +1465,7 @@ def __init__( image, "sleep", "infinity", # no fixed lifetime — idle reaper handles cleanup ] - logger.debug(f"Starting container: {' '.join(run_cmd)}") + logger.debug("Starting container: %s", ' '.join(run_cmd)) try: result = subprocess.run( run_cmd, @@ -1494,7 +1494,7 @@ def __init__( ) raise self._container_id = result.stdout.strip() - logger.info(f"Started container {container_name} ({self._container_id[:12]})") + logger.info("Started container %s (%s)", container_name, self._container_id[:12]) # Build the init-time env forwarding args used to seed the snapshot. self._init_env_args = self._build_init_env_args() diff --git a/tools/file_tools.py b/tools/file_tools.py index 9a9590c9f7f5..ad3050efd5ab 100644 --- a/tools/file_tools.py +++ b/tools/file_tools.py @@ -872,6 +872,8 @@ def _reset_patch_failures(task_id: str, resolved_paths: list) -> None: _READ_HISTORY_CAP = 500 # set; used only by get_read_files_summary _DEDUP_CAP = 1000 # dict; skip-identical-reread guard _READ_TIMESTAMPS_CAP = 1000 # dict; external-edit detection for write/patch +_NOT_FOUND_CAP = 500 # dict; per-task negative-result cache for missing paths +_NOT_FOUND_TTL_SECONDS = 60.0 # short TTL — a path that didn't exist may be created soon _READ_DEDUP_STATUS_MESSAGE = ( "File unchanged since last read. The content from " "the earlier read_file result in this conversation is " @@ -929,6 +931,79 @@ def _cap_read_tracker_data(task_data: dict) -> None: except (StopIteration, KeyError): break + nf = task_data.get("not_found") + if nf is not None and len(nf) > _NOT_FOUND_CAP: + excess = len(nf) - _NOT_FOUND_CAP + for _ in range(excess): + try: + nf.pop(next(iter(nf))) + except (StopIteration, KeyError): + break + + +def _check_not_found_cache(op: str, resolved_str: str, task_id: str) -> str | None: + """Return cached not-found JSON for *(op, resolved_str)* if still fresh. + + Skips the expensive subprocess + suggestion walk when the model retries + the same missing path. Observed in agent.log: a single typo'd path was + retried 13 times — each retry forked a shell to walk the parent directory + and score similar names. + + *op* is "read" or "search" — kept separate because the two callers return + different error JSON shapes ("File not found:" vs "Path not found:"). + + Eviction: TTL or write_file/patch on the path (see invalidate_for_path). + """ + import os as _os + import time + with _read_tracker_lock: + task_data = _read_tracker.get(task_id) + if not task_data: + return None + nf = task_data.get("not_found") + if not nf: + return None + entry = nf.get((op, resolved_str)) + if entry is None: + return None + ts, cached_json = entry + if time.monotonic() - ts > _NOT_FOUND_TTL_SECONDS: + nf.pop((op, resolved_str), None) + return None + # Existence guard: the path may have been created since we cached the + # miss — by a terminal command, another agent, or any external process + # (write_file/patch invalidate explicitly, but they're not the only + # writers). The agent pattern "check file → create it → read it" is + # common; serving a stale miss for up to the TTL breaks it. One stat is + # ~free next to the subprocess walk we're skipping. + # + # The stat runs OUTSIDE _read_tracker_lock (matching the dedup mtime + # check below in read_file_tool): the lock is global across all tasks, + # and a hung stat on a dead network mount must not stall every other + # task's read/search bookkeeping. + if _os.path.exists(resolved_str): + with _read_tracker_lock: + task_data = _read_tracker.get(task_id) + nf = task_data.get("not_found") if task_data else None + if nf: + nf.pop((op, resolved_str), None) + return None + return cached_json + + +def _record_not_found(op: str, resolved_str: str, task_id: str, error_json: str) -> None: + """Cache a not-found error so the next *op* call for *resolved_str* skips I/O.""" + import time + with _read_tracker_lock: + task_data = _read_tracker.setdefault(task_id, { + "last_key": None, "consecutive": 0, + "read_history": set(), "dedup": {}, + "dedup_hits": {}, "read_timestamps": {}, + }) + nf = task_data.setdefault("not_found", {}) + nf[(op, resolved_str)] = (time.monotonic(), error_json) + _cap_read_tracker_data(task_data) + def _is_internal_file_status_text(content: str) -> bool: """Return True when content looks like an internal file-tool status, not real file bytes. @@ -1282,6 +1357,15 @@ def read_file_tool(path: str, offset: int = 1, limit: int = 500, task_id: str = if block_error: return tool_error(block_error) + # ── Negative-result cache ───────────────────────────────────── + # If we already discovered this path doesn't exist (within TTL), + # return the cached error without spawning the subprocess + + # similar-files walk. Cleared by write_file/patch on the same path. + resolved_str_for_neg = str(_resolved) + cached_not_found = _check_not_found_cache("read", resolved_str_for_neg, task_id) + if cached_not_found is not None: + return cached_not_found + # ── Dedup check ─────────────────────────────────────────────── # If we already read this exact (path, offset, limit) and the # file hasn't been modified since, return a lightweight stub @@ -1345,6 +1429,20 @@ def read_file_tool(path: str, offset: int = 1, limit: int = 500, task_id: str = result = file_ops.read_file(path, offset, limit) result_dict = result.to_dict() + # ── Populate negative-result cache on not-found ─────────────── + # _suggest_similar_files returns ReadResult(error="File not found: .."). + # Cache the JSON we'd return so a retry skips the parent-dir walk. + # Deliberately NO early return: on upstream, error results flow + # through the tracking block below (consecutive-loop detection, + # dedup bookkeeping via the resolved path) and the normal exit — + # short-circuiting here changes that behavior (and broke a real + # test interaction). Serving from the cache (above) is the + # optimization; recording must stay side-effect-identical. + _err = result_dict.get("error") or "" + if isinstance(_err, str) and _err.startswith("File not found:"): + _not_found_json = json.dumps(result_dict, ensure_ascii=False) + _record_not_found("read", resolved_str_for_neg, task_id, _not_found_json) + # ── Character-count guard ───────────────────────────────────── # We're model-agnostic so we can't count tokens; characters are # the best proxy we have. If the read produced an unreasonable @@ -1520,6 +1618,15 @@ def notify_other_tool_call(task_id: str = "default"): # progress, so clear per-key dedup hit counters too. if "dedup_hits" in task_data: task_data["dedup_hits"].clear() + # Any other tool (terminal, delegate, ...) may have created a + # previously-missing path — a cached miss is no longer + # trustworthy. The serve-side existence guard in + # _check_not_found_cache already covers this, but clearing + # here keeps the cache honest and covers exotic cases the + # stat can't (e.g. permission flips). + nf = task_data.get("not_found") + if nf: + nf.clear() def _invalidate_dedup_for_path(filepath: str, task_id: str) -> None: @@ -1544,12 +1651,18 @@ def _invalidate_dedup_for_path(filepath: str, task_id: str) -> None: if task_data is None: return dedup = task_data.get("dedup") - if not dedup: - return - # Collect keys to remove (can't mutate dict during iteration). - stale_keys = [k for k in dedup if k[0] == resolved] - for k in stale_keys: - del dedup[k] + if dedup: + # Collect keys to remove (can't mutate dict during iteration). + stale_keys = [k for k in dedup if k[0] == resolved] + for k in stale_keys: + del dedup[k] + # Also evict from the negative-result cache: a write_file that + # creates the path means subsequent reads (or searches under it) + # must hit disk. + nf = task_data.get("not_found") + if nf: + nf.pop(("read", resolved), None) + nf.pop(("search", resolved), None) def _update_read_timestamp(filepath: str, task_id: str) -> None: @@ -1974,6 +2087,19 @@ def search_tool(pattern: str, target: str = "content", path: str = ".", if block_error: return tool_error(block_error) + # ── Negative-result cache ───────────────────────────────────── + # Search returns "Path not found: " when the search root + # doesn't exist. The error path also lists the parent directory + # (file_operations.py:1402) — expensive to repeat. Cache so the + # next call to a known-missing root skips both shells. + try: + resolved_search_path = str(_resolve_path_for_task(path, task_id)) + except (OSError, ValueError): + resolved_search_path = path + cached_search_nf = _check_not_found_cache("search", resolved_search_path, task_id) + if cached_search_nf is not None: + return cached_search_nf + file_ops = _get_file_ops(task_id) result = file_ops.search( pattern=pattern, path=path, target=target, file_glob=file_glob, @@ -1992,6 +2118,14 @@ def search_tool(pattern: str, target: str = "content", path: str = ".", "token, cache, or secret-bearing environment files." ) + # Populate negative cache when search root was missing. No early + # return — same rationale as the read path: error results keep + # flowing through the consecutive-search bookkeeping below. + _search_err = result_dict.get("error") or "" + if isinstance(_search_err, str) and _search_err.startswith("Path not found:"): + _search_nf_json = json.dumps(result_dict, ensure_ascii=False) + _record_not_found("search", resolved_search_path, task_id, _search_nf_json) + if count >= 3: result_dict["_warning"] = ( f"You have run this exact search {count} times consecutively. " diff --git a/tools/managed_tool_gateway.py b/tools/managed_tool_gateway.py index af7f8f69748d..c46a48975bb4 100644 --- a/tools/managed_tool_gateway.py +++ b/tools/managed_tool_gateway.py @@ -73,6 +73,28 @@ def _access_token_is_expiring(expires_at: object, skew_seconds: int) -> bool: return remaining <= max(0, int(skew_seconds)) +def _read_user_token_override() -> Optional[str]: + """Read the TOOL_GATEWAY_USER_TOKEN env override through the secret scope. + + Availability scans run both inside agent turns (scope installed) and in + unscoped CLI paths, so this uses the Slack pattern: honor the scope's + verdict when installed (a scoped miss does NOT borrow the process env + under multiplex), fall back to ``os.environ`` only when unscoped. + """ + try: + from agent.secret_scope import UnscopedSecretError, get_secret + + try: + explicit = get_secret("TOOL_GATEWAY_USER_TOKEN") + except UnscopedSecretError: + explicit = os.getenv("TOOL_GATEWAY_USER_TOKEN") + except Exception: + explicit = os.getenv("TOOL_GATEWAY_USER_TOKEN") + if isinstance(explicit, str) and explicit.strip(): + return explicit.strip() + return None + + def peek_nous_access_token() -> Optional[str]: """Cheap probe for a Nous gateway token without triggering refresh. @@ -83,9 +105,9 @@ def peek_nous_access_token() -> Optional[str]: network calls. Truthful refresh handling stays in request/session paths that call :func:`read_nous_access_token`. """ - explicit = os.getenv("TOOL_GATEWAY_USER_TOKEN") - if isinstance(explicit, str) and explicit.strip(): - return explicit.strip() + explicit = _read_user_token_override() + if explicit: + return explicit nous_provider = _read_nous_provider_state() or {} access_token = nous_provider.get("access_token") @@ -96,9 +118,9 @@ def peek_nous_access_token() -> Optional[str]: def read_nous_access_token() -> Optional[str]: """Read a Nous Subscriber OAuth access token from auth store or env override.""" - explicit = os.getenv("TOOL_GATEWAY_USER_TOKEN") - if isinstance(explicit, str) and explicit.strip(): - return explicit.strip() + explicit = _read_user_token_override() + if explicit: + return explicit nous_provider = _read_nous_provider_state() or {} cached_token = peek_nous_access_token() diff --git a/tools/openrouter_client.py b/tools/openrouter_client.py index 0637a7db0ded..9c857076c875 100644 --- a/tools/openrouter_client.py +++ b/tools/openrouter_client.py @@ -29,5 +29,19 @@ def get_async_client(): def check_api_key() -> bool: - """Check whether the OpenRouter API key is present.""" + """Check whether the OpenRouter API key is present. + + Scope-aware (Slack pattern): tool paths run inside an installed profile + secret scope, whose verdict is authoritative under multiplex; unscoped + CLI probes keep the legacy env read. + """ + try: + from agent.secret_scope import UnscopedSecretError, get_secret + + try: + return bool(get_secret("OPENROUTER_API_KEY")) + except UnscopedSecretError: + pass + except Exception: + pass return bool(os.getenv("OPENROUTER_API_KEY")) diff --git a/tools/send_message_tool.py b/tools/send_message_tool.py index 4e57dee104c5..2853c6275c49 100644 --- a/tools/send_message_tool.py +++ b/tools/send_message_tool.py @@ -2006,10 +2006,15 @@ async def _send_qqbot(pconfig, chat_id, message): except ImportError: return _error("QQBot direct send requires httpx. Run: pip install httpx") + # Resolve credential fallbacks through the profile secret scope (with the + # plain-environ fallback for unscoped single-profile runs) so a multiplex + # profile's direct send never borrows another profile's QQ credentials. + from gateway.config import _getenv + extra = pconfig.extra or {} - appid = extra.get("app_id") or os.getenv("QQ_APP_ID", "") + appid = extra.get("app_id") or _getenv("QQ_APP_ID", "") secret = (pconfig.token or extra.get("client_secret") - or os.getenv("QQ_CLIENT_SECRET", "")) + or _getenv("QQ_CLIENT_SECRET", "")) if not appid or not secret: return _error("QQBot: QQ_APP_ID / QQ_CLIENT_SECRET not configured.") diff --git a/tools/skills_hub.py b/tools/skills_hub.py index a193895ca5cf..0316fee9d04b 100644 --- a/tools/skills_hub.py +++ b/tools/skills_hub.py @@ -382,8 +382,9 @@ def _resolve_token(self) -> Optional[str]: if self._cached_method != "github-app" or time.time() < self._app_token_expiry: return self._cached_token - # 1. Environment variable - token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") + # 1. Environment variable (profile-scoped under a multiplexed gateway) + from agent.secret_scope import get_secret + token = get_secret("GITHUB_TOKEN") or get_secret("GH_TOKEN") if token: self._cached_token = token self._cached_method = "pat" @@ -424,9 +425,10 @@ def _try_gh_cli(self) -> Optional[str]: def _try_github_app(self) -> Optional[str]: """Try GitHub App JWT authentication if credentials are configured.""" - app_id = os.environ.get("GITHUB_APP_ID") - key_path = os.environ.get("GITHUB_APP_PRIVATE_KEY_PATH") - installation_id = os.environ.get("GITHUB_APP_INSTALLATION_ID") + from agent.secret_scope import get_secret + app_id = get_secret("GITHUB_APP_ID") + key_path = get_secret("GITHUB_APP_PRIVATE_KEY_PATH") + installation_id = get_secret("GITHUB_APP_INSTALLATION_ID") if not all([app_id, key_path, installation_id]): return None @@ -462,7 +464,7 @@ def _try_github_app(self) -> Optional[str]: if resp.status_code == 201: return resp.json().get("token") except Exception as e: - logger.debug(f"GitHub App auth failed: {e}") + logger.debug("GitHub App auth failed: %s", e) return None @@ -623,7 +625,7 @@ def search(self, query: str, limit: int = 10) -> List[SkillMeta]: if query_lower in searchable: results.append(skill) except Exception as e: - logger.debug(f"Failed to search {tap['repo']}: {e}") + logger.debug("Failed to search %s: %s", tap['repo'], e) continue # Deduplicate by identifier, preferring higher trust levels. diff --git a/tools/terminal_tool.py b/tools/terminal_tool.py index 7f9141ff8a9e..1768f80aed3c 100644 --- a/tools/terminal_tool.py +++ b/tools/terminal_tool.py @@ -159,10 +159,12 @@ def _check_vercel_sandbox_requirements(config: dict[str, Any]) -> bool: ) return False - has_oidc = bool(os.getenv("VERCEL_OIDC_TOKEN")) - has_token = bool(os.getenv("VERCEL_TOKEN")) - has_project = bool(os.getenv("VERCEL_PROJECT_ID")) - has_team = bool(os.getenv("VERCEL_TEAM_ID")) + from agent.secret_scope import get_secret + + has_oidc = bool(get_secret("VERCEL_OIDC_TOKEN")) + has_token = bool(get_secret("VERCEL_TOKEN")) + has_project = bool(get_secret("VERCEL_PROJECT_ID")) + has_team = bool(get_secret("VERCEL_TEAM_ID")) if has_oidc: return True @@ -187,8 +189,24 @@ def _check_vercel_sandbox_requirements(config: dict[str, Any]) -> bool: return False +# Cache for disk usage warning to avoid full rglob scan on every call. +# The check is advisory-only — staleness for up to 5 minutes is acceptable. +_disk_usage_cache: dict = {"timestamp": 0.0, "result": False} +_DISK_USAGE_CACHE_TTL = 300.0 # seconds + + def _check_disk_usage_warning(): - """Check if total disk usage exceeds warning threshold.""" + """Check if total disk usage exceeds warning threshold. + + Result is cached for :data:`_DISK_USAGE_CACHE_TTL` seconds (default: + 5 minutes) to avoid an expensive recursive filesystem scan on every + terminal command. The check is advisory-only so a stale result is + harmless. + """ + import time as _time_mod + now = _time_mod.monotonic() + if now - _disk_usage_cache["timestamp"] < _DISK_USAGE_CACHE_TTL: + return _disk_usage_cache["result"] try: scratch_dir = _get_scratch_dir() @@ -205,14 +223,16 @@ def _check_disk_usage_warning(): total_gb = total_bytes / (1024 ** 3) - if total_gb > DISK_USAGE_WARNING_THRESHOLD_GB: + exceeded = total_gb > DISK_USAGE_WARNING_THRESHOLD_GB + if exceeded: logger.warning("Disk usage (%.1fGB) exceeds threshold (%.0fGB). Consider running cleanup_all_environments().", total_gb, DISK_USAGE_WARNING_THRESHOLD_GB) - return True - - return False + _disk_usage_cache["timestamp"] = _time_mod.monotonic() + _disk_usage_cache["result"] = exceeded + return exceeded except Exception as e: logger.debug("Disk usage warning check failed: %s", e, exc_info=True) + # Don't update cache on error so the next call retries. return False @@ -977,9 +997,21 @@ def _transform_sudo_command(command: str | None) -> tuple[str | None, str | None if sudo_count == 0: return command, None - has_configured_password = "SUDO_PASSWORD" in os.environ + # Scope-aware read (Slack pattern): under multiplex the process env may + # hold another profile's SUDO_PASSWORD, so honor the installed scope's + # verdict; unscoped callers keep the legacy os.environ read. + try: + from agent.secret_scope import UnscopedSecretError, get_secret + + try: + _configured_password = get_secret("SUDO_PASSWORD") + except UnscopedSecretError: + _configured_password = os.environ.get("SUDO_PASSWORD") + except Exception: + _configured_password = os.environ.get("SUDO_PASSWORD") + has_configured_password = _configured_password is not None sudo_password = ( - os.environ.get("SUDO_PASSWORD", "") + _configured_password if has_configured_password else _get_cached_sudo_password() ) @@ -3177,7 +3209,8 @@ def check_terminal_requirements() -> bool: elif env_type == "daytona": from daytona import Daytona # noqa: F401 — SDK presence check - return os.getenv("DAYTONA_API_KEY") is not None + from agent.secret_scope import get_secret + return get_secret("DAYTONA_API_KEY") is not None else: logger.error( diff --git a/tools/tirith_security.py b/tools/tirith_security.py index 2ea2eec749d9..a284c6d4007c 100644 --- a/tools/tirith_security.py +++ b/tools/tirith_security.py @@ -283,7 +283,8 @@ def is_platform_supported() -> bool: def _download_file(url: str, dest: str, timeout: int = 10): """Download a URL to a local file.""" req = urllib.request.Request(url) - token = os.getenv("GITHUB_TOKEN") + from agent.secret_scope import get_secret + token = get_secret("GITHUB_TOKEN") if token: req.add_header("Authorization", f"token {token}") with urllib.request.urlopen(req, timeout=timeout) as resp, open(dest, "wb") as f: diff --git a/tools/tool_backend_helpers.py b/tools/tool_backend_helpers.py index de1fec7e8c5b..b00814c0ad23 100644 --- a/tools/tool_backend_helpers.py +++ b/tools/tool_backend_helpers.py @@ -298,7 +298,7 @@ def fal_key_is_configured() -> bool: checks and CLI setup-time checks agree. A whitespace-only value is treated as unset everywhere. """ - value = os.getenv("FAL_KEY") + value = _scoped_credential("FAL_KEY") or None if value is None: # Fall back to the .env file for CLI paths that may run before # dotenv is loaded into os.environ. diff --git a/tools/xai_http.py b/tools/xai_http.py index 8ef0b856302d..5350a27d76f7 100644 --- a/tools/xai_http.py +++ b/tools/xai_http.py @@ -36,8 +36,14 @@ def has_xai_credentials() -> bool: other availability scans. Truthful refresh + expiry handling happens in ``search()`` (or whichever caller actually makes the request). """ - if os.environ.get("XAI_API_KEY", "").strip(): - return True + try: + from agent.secret_scope import get_secret + except ImportError: # pragma: no cover — secret_scope is in-repo + if os.environ.get("XAI_API_KEY", "").strip(): + return True + else: + if (get_secret("XAI_API_KEY", "") or "").strip(): + return True try: from hermes_constants import get_hermes_home @@ -79,13 +85,11 @@ def get_env_value(name: str, default=None): """ try: from hermes_cli.config import get_env_value as _hermes_get_env_value + except ImportError: + return os.environ.get(name, default) - value = _hermes_get_env_value(name) - if value is not None: - return value - except Exception: - pass - return os.environ.get(name, default) + value = _hermes_get_env_value(name) + return value if value is not None else default def hermes_xai_user_agent() -> str: diff --git a/trajectory_compressor.py b/trajectory_compressor.py index 83248983a2ca..8feb4a5157cc 100644 --- a/trajectory_compressor.py +++ b/trajectory_compressor.py @@ -663,7 +663,7 @@ def _generate_summary(self, content: str, metrics: TrajectoryMetrics) -> str: except Exception as e: metrics.summarization_errors += 1 - self.logger.warning(f"Summarization attempt {attempt + 1} failed: {e}") + self.logger.warning("Summarization attempt %d failed: %s", attempt + 1, e) if attempt < self.config.max_retries - 1: time.sleep(jittered_backoff(attempt + 1, base_delay=self.config.retry_delay, max_delay=30.0)) @@ -732,7 +732,7 @@ async def _generate_summary_async(self, content: str, metrics: TrajectoryMetrics except Exception as e: metrics.summarization_errors += 1 - self.logger.warning(f"Summarization attempt {attempt + 1} failed: {e}") + self.logger.warning("Summarization attempt %d failed: %s", attempt + 1, e) if attempt < self.config.max_retries - 1: await asyncio.sleep(jittered_backoff(attempt + 1, base_delay=self.config.retry_delay, max_delay=30.0)) @@ -1087,7 +1087,7 @@ async def _process_directory_async(self, input_dir: Path, output_dir: Path): jsonl_files = sorted(input_dir.glob("*.jsonl")) if not jsonl_files: - self.logger.warning(f"No JSONL files found in {input_dir}") + self.logger.warning("No JSONL files found in %s", input_dir) return # Load ALL entries from all files @@ -1103,7 +1103,7 @@ async def _process_directory_async(self, input_dir: Path, output_dir: Path): entry = json.loads(line) all_entries.append((file_path, line_num, entry)) except json.JSONDecodeError as e: - self.logger.warning(f"Skipping invalid JSON at {file_path}:{line_num}: {e}") + self.logger.warning("Skipping invalid JSON at %s:%s: %s", file_path, line_num, e) total_entries = len(all_entries) @@ -1172,7 +1172,7 @@ async def process_single(file_path: Path, entry_idx: int, entry: Dict, ) except asyncio.TimeoutError: - self.logger.warning(f"Timeout processing entry from {file_path}:{entry_idx} (>{self.config.per_trajectory_timeout}s)") + self.logger.warning("Timeout processing entry from %s:%s (>%ss)", file_path, entry_idx, self.config.per_trajectory_timeout) async with progress_lock: self.aggregate_metrics.trajectories_failed += 1 @@ -1188,7 +1188,7 @@ async def process_single(file_path: Path, entry_idx: int, entry: Dict, results[file_path][entry_idx] = None except Exception as e: - self.logger.error(f"Error processing entry from {file_path}:{entry_idx}: {e}") + self.logger.error("Error processing entry from %s:%s: %s", file_path, entry_idx, e) async with progress_lock: self.aggregate_metrics.trajectories_failed += 1 diff --git a/tui_gateway/compute_host.py b/tui_gateway/compute_host.py index 83aaf740b924..1f255533bd7b 100644 --- a/tui_gateway/compute_host.py +++ b/tui_gateway/compute_host.py @@ -21,6 +21,8 @@ from pathlib import Path from typing import Any, Callable +from agent.interrupt_compat import request_hard_interrupt + def now_ns() -> int: return time.perf_counter_ns() @@ -37,7 +39,7 @@ class SpikeAgent: def clear_interrupt(self) -> None: self._interrupt.clear() - def interrupt(self) -> None: + def interrupt(self, *, hard_cancel: bool = False) -> None: self._interrupt.set() def run_conversation( @@ -257,7 +259,7 @@ def _handle_interrupt(self, frame: dict[str, Any]) -> None: sid = str(frame.get("sid") or "") spike = self._sessions.get(sid) if spike is not None: - spike.agent.interrupt() + request_hard_interrupt(spike.agent) self.emit( { "type": "interrupt.ack", @@ -276,8 +278,8 @@ def _handle_interrupt(self, frame: dict[str, Any]) -> None: self.emit({"type": "interrupt.ack", "sid": sid, "request_id": frame.get("request_id"), "applied": False}) return agent = session.get("agent") - if agent is not None and hasattr(agent, "interrupt"): - agent.interrupt() + if agent is not None: + request_hard_interrupt(agent) with session.get("history_lock", threading.Lock()): session["_turn_cancel_requested"] = True session["queued_prompt"] = None diff --git a/tui_gateway/methods_session.py b/tui_gateway/methods_session.py index 44ef31ca82d5..75e473426b87 100644 --- a/tui_gateway/methods_session.py +++ b/tui_gateway/methods_session.py @@ -2656,6 +2656,16 @@ def _(rid, params: dict) -> dict: home_token = ( set_hermes_home_override(parent_home) if parent_home else None ) + # The home override alone only moves config/skills/memory; credentials + # resolve through get_secret(), which without a scope falls through to + # process os.environ — the LAUNCH profile's .env. Install the parent's + # secret scope for the build, exactly as session.create/resume do + # (#67605), so the branched agent authenticates as its own profile. + secret_token = ( + set_secret_scope(build_profile_secret_scope(Path(parent_home))) + if parent_home + else None + ) try: tokens = _set_session_context(new_key) try: @@ -2680,6 +2690,8 @@ def _(rid, params: dict) -> dict: profile_home=parent_home, ) finally: + if secret_token is not None: + reset_secret_scope(secret_token) if home_token is not None: reset_hermes_home_override(home_token) if new_sid in _sessions: @@ -2749,8 +2761,10 @@ def _(rid, params: dict) -> dict: session["queued_prompt"] = None session.pop("queued_prompts", None) session["_queued_prompt_generation"] = int(session.get("_queued_prompt_generation", 0)) + 1 - if should_interrupt and hasattr(session["agent"], "interrupt"): - session["agent"].interrupt() + if should_interrupt: + from agent.interrupt_compat import request_hard_interrupt + + request_hard_interrupt(session["agent"]) if not run_thread_alive: with session["history_lock"]: if session.get("running"): diff --git a/tui_gateway/methods_tools.py b/tui_gateway/methods_tools.py index 53be0136bcc5..5896f1281c77 100644 --- a/tui_gateway/methods_tools.py +++ b/tui_gateway/methods_tools.py @@ -1384,7 +1384,9 @@ def _(rid, params: dict) -> dict: try: cfg = _load_cfg() model = _resolve_model() - api_key = os.environ.get("HERMES_API_KEY", "") or cfg.get("api_key", "") + from agent.secret_scope import get_secret + + api_key = get_secret("HERMES_API_KEY", "") or cfg.get("api_key", "") masked = f"****{api_key[-4:]}" if len(api_key) > 4 else "(not set)" base_url = os.environ.get("HERMES_BASE_URL", "") or cfg.get("base_url", "") diff --git a/tui_gateway/server.py b/tui_gateway/server.py index 029d8ae72367..015ec90df3c2 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -1054,6 +1054,21 @@ def _reap_idle_sessions() -> None: _close_session_by_id(sid, end_reason="idle_timeout") _enforce_session_cap() _reclaim_orphaned_leases() + # Periodic heap release for long-lived gateway processes. Even when no + # session is reaped, Python's generational GC rarely runs gen2 collection + # under steady-state allocation, and glibc retains freed pages as RSS. + # Calling trim_memory here ensures every reaper scan (default every 5 min) + # returns releasable pages, preventing unbounded RSS growth over days/weeks. + try: + from hermes_cli.mem_trim import trim_memory + + trim_memory(reason="idle reaper periodic trim") + except Exception as exc: + # debug, not warning — persistent failure would repeat every reaper + # scan (300s) forever; sibling failure branches log at debug. + logger.debug( + "idle reaper memory trim failed: %s: %s", type(exc).__name__, exc + ) def _reclaim_orphaned_leases() -> None: @@ -9826,6 +9841,23 @@ def _interim_assistant_cb(text: str, *, already_streamed: bool = False) -> None: ) _emit("error", sid, {"message": str(e)}) finally: + # Drop both local snapshots of the pre-turn history before asking + # glibc to return pages. session["history"] already points at the + # new/pruned result; retaining either list defeats this trim. + history.clear() + local_run_kwargs = locals().get("run_kwargs") + if isinstance(local_run_kwargs, dict): + local_run_kwargs.clear() + + # Run while any profile-specific HERMES_HOME override is still active + # so context.memory_trim is resolved from the session's own config. + try: + from hermes_cli.mem_trim import trim_memory + + trim_memory(reason="tui turn completion") + except Exception: + logger.debug("post-turn memory trim failed", exc_info=True) + if thinking_started: # Kill the ambient thinking sound the moment the turn ends — # error and success paths both land here. diff --git a/tui_gateway/slash_worker.py b/tui_gateway/slash_worker.py index 5b75dbd16565..7066a62f434d 100644 --- a/tui_gateway/slash_worker.py +++ b/tui_gateway/slash_worker.py @@ -20,6 +20,7 @@ import contextlib import io import json +import logging import os import sys import threading @@ -48,6 +49,7 @@ def _env_float(name: str, default: float) -> float: _WATCHDOG_POLL_S = max(0.05, _env_float("HERMES_SLASH_WATCHDOG_POLL_S", 2.0)) _ORPHAN_GRACE_S = max(0.0, _env_float("HERMES_SLASH_WATCHDOG_GRACE_S", 5.0)) _in_flight = threading.Event() # set while a command is executing +logger = logging.getLogger(__name__) def _is_orphaned(original_ppid, getppid=os.getppid) -> bool: @@ -173,6 +175,21 @@ def _sw_log(reason: str) -> None: sys.stdout.flush() finally: _in_flight.clear() + # Workers persist for the TUI session, so release allocator pages at + # the same command boundary as other long-lived gateway processes. + # trim_memory's shared cooldown coalesces this with nearby activity. + try: + from hermes_cli.mem_trim import trim_memory + + trim_memory(reason="slash worker command completion") + except Exception as exc: + # debug, not warning — a persistent failure would repeat on + # every slash command forever. + logger.debug( + "slash worker memory trim failed: %s: %s", + type(exc).__name__, + exc, + ) if __name__ == "__main__":