From 5c930012aff5a1a84eb1733dd0120ca24c35db49 Mon Sep 17 00:00:00 2001 From: Varun Garde <8184926+vgarde@users.noreply.github.com> Date: Sun, 9 Aug 2026 21:54:25 -0500 Subject: [PATCH 1/2] fix(security): redact reasoning display and session exports; cover Google OAuth client secrets (#20785) --- agent/redact.py | 3 ++- cli.py | 4 ++++ gateway/run.py | 4 ++++ hermes_cli/session_export.py | 10 ++++++++-- 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/agent/redact.py b/agent/redact.py index b5dca332c4ade..6ae336a09155e 100644 --- a/agent/redact.py +++ b/agent/redact.py @@ -90,6 +90,7 @@ r"pplx-[A-Za-z0-9]{10,}", # Perplexity r"fal_[A-Za-z0-9_-]{10,}", # Fal.ai r"fc-[A-Za-z0-9]{10,}", # Firecrawl + r"GOCSPX-[A-Za-z0-9_-]{10,}", # Google OAuth client secret r"bb_live_[A-Za-z0-9_-]{10,}", # BrowserBase r"gAAAA[A-Za-z0-9_=-]{20,}", # Codex encrypted tokens r"AKIA[A-Z0-9]{16}", # AWS Access Key ID @@ -314,7 +315,7 @@ def _key_has_secret_keyword(key: str) -> bool: return False # JSON field patterns: "apiKey": "value", "token": "value", etc. -_JSON_KEY_NAMES = r"(?:api_?[Kk]ey|token|secret|password|access_token|refresh_token|auth_token|bearer|secret_value|raw_secret|secret_input|key_material)" +_JSON_KEY_NAMES = r"(?:api_?[Kk]ey|secret_key|client_secret|app_secret|token|secret|password|access_token|refresh_token|auth_token|bearer|secret_value|raw_secret|secret_input|key_material)" _JSON_FIELD_RE = re.compile( rf'("{_JSON_KEY_NAMES}")\s*:\s*"([^"]+)"', re.IGNORECASE, diff --git a/cli.py b/cli.py index 541d112a52772..f0a4f10a94634 100644 --- a/cli.py +++ b/cli.py @@ -14413,6 +14413,10 @@ def run_agent(): display_reasoning += f"\n{_DIM} ... ({len(lines) - 10} more lines — /reasoning full to show){_RST}" else: display_reasoning = reasoning.strip() + # Scrub credential patterns from scratch thinking before + # it renders (#20785) — reasoning is display-only text. + from agent.redact import redact_sensitive_text + display_reasoning = redact_sensitive_text(display_reasoning) _cprint(f"\n{r_top}\n{_DIM}{display_reasoning}{_RST}\n{r_bot}") if response and not response_previewed: diff --git a/gateway/run.py b/gateway/run.py index 52ae139c5c24e..01169cac5022a 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -17783,6 +17783,10 @@ async def _handle_message_with_agent(self, event, source, _quick_key: str, run_g display_reasoning += f"\n_... ({len(lines) - 15} more lines)_" else: display_reasoning = last_reasoning.strip() + # Never let reasoning/thinking scratch text carry + # credential values into user-facing chat (#20785). + from agent.redact import redact_sensitive_text + display_reasoning = redact_sensitive_text(display_reasoning) # Render style is per-platform: Discord defaults to "-# " # subtext (native small grey metadata text); other # platforms keep the fenced code block. diff --git a/hermes_cli/session_export.py b/hermes_cli/session_export.py index 031c21034ede1..6f7baf81d0c89 100644 --- a/hermes_cli/session_export.py +++ b/hermes_cli/session_export.py @@ -56,8 +56,14 @@ def render_sessions_export( export_only = normalize_export_only(only) if export_format == "jsonl": - return _render_jsonl(session_list, only=export_only) - return _render_markdown(session_list, only=export_only) + rendered = _render_jsonl(session_list, only=export_only) + else: + rendered = _render_markdown(session_list, only=export_only) + # Never let exported transcripts carry credential values (#20785). + # Redaction replaces matches with a plain placeholder, so JSON/MD + # structure stays intact. + from agent.redact import redact_sensitive_text + return redact_sensitive_text(rendered) def export_record_count( From 93e57d87508877f2560aad63bda6aea46f43d4f9 Mon Sep 17 00:00:00 2001 From: Varun Garde <8184926+vgarde@users.noreply.github.com> Date: Tue, 18 Aug 2026 11:00:09 -0500 Subject: [PATCH 2/2] fix(security): redact structured exports pre-serialization; force egress redaction (#20785) --- agent/redact.py | 46 ++++++++++++++++++ cli.py | 2 +- gateway/run.py | 2 +- hermes_cli/session_export.py | 17 +++++-- tests/agent/test_redact.py | 51 +++++++++++++++++++- tests/hermes_cli/test_session_export.py | 64 +++++++++++++++++++++++++ 6 files changed, 175 insertions(+), 7 deletions(-) diff --git a/agent/redact.py b/agent/redact.py index 6ae336a09155e..919651ed3b3a7 100644 --- a/agent/redact.py +++ b/agent/redact.py @@ -11,6 +11,7 @@ import os import re import shlex +from typing import Any from urllib.parse import unquote_plus # Basenames treated as ``.env`` files by _command_reads_env_file. Imported @@ -1008,6 +1009,51 @@ def _redact_phone(m): return text +def redact_structured(value: Any, *, force: bool = True) -> Any: + """Deep-redact credential values inside a JSON-like structure. + + Walks dicts/lists/tuples recursively and: + + - runs every string through :func:`redact_sensitive_text` so prefixed + credentials (``sk-``, ``GOCSPX-``, …) are masked anywhere they appear; + - for a dict entry whose *key* names a credential field (``client_secret``, + ``apiKey``, ``token``, ``password``, …), masks the value entirely even + when it has no recognizable prefix — the key context alone identifies + it as a secret. This is what text-only redaction misses for structured + content: once serialized to JSONL the key is quoted (``\\"client_secret\\"``) + and the value is opaque, so no pattern fires (issue #20785 follow-up). + + Non-string scalars and empty strings pass through unchanged so structure + and JSON types survive (the output stays parseable). + + ``force=True`` is the default because this is an egress boundary: exports + must never carry credential values even when ``security.redact_secrets`` + is disabled globally. + """ + if isinstance(value, str): + if not value: + return value + return redact_sensitive_text(value, force=force) + if isinstance(value, dict): + out: Dict[Any, Any] = {} + for key, item in value.items(): + if isinstance(key, str) and _key_has_secret_keyword(key): + if isinstance(item, str) and item: + out[key] = _mask_token_nonreusable(item) + elif isinstance(item, (dict, list)): + out[key] = redact_structured(item, force=force) + else: + out[key] = item + else: + out[key] = redact_structured(item, force=force) + return out + if isinstance(value, list): + return [redact_structured(item, force=force) for item in value] + if isinstance(value, tuple): + return tuple(redact_structured(item, force=force) for item in value) + return value + + # Commands whose stdout is an environment-variable dump (KEY=value lines), # NOT source code. For these, terminal-output redaction must run the # ENV-assignment pass (code_file=False) so opaque tokens with no recognized diff --git a/cli.py b/cli.py index f0a4f10a94634..5707bfe8b0213 100644 --- a/cli.py +++ b/cli.py @@ -14416,7 +14416,7 @@ def run_agent(): # Scrub credential patterns from scratch thinking before # it renders (#20785) — reasoning is display-only text. from agent.redact import redact_sensitive_text - display_reasoning = redact_sensitive_text(display_reasoning) + display_reasoning = redact_sensitive_text(display_reasoning, force=True) _cprint(f"\n{r_top}\n{_DIM}{display_reasoning}{_RST}\n{r_bot}") if response and not response_previewed: diff --git a/gateway/run.py b/gateway/run.py index 01169cac5022a..a30ad490de297 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -17786,7 +17786,7 @@ async def _handle_message_with_agent(self, event, source, _quick_key: str, run_g # Never let reasoning/thinking scratch text carry # credential values into user-facing chat (#20785). from agent.redact import redact_sensitive_text - display_reasoning = redact_sensitive_text(display_reasoning) + display_reasoning = redact_sensitive_text(display_reasoning, force=True) # Render style is per-platform: Discord defaults to "-# " # subtext (native small grey metadata text); other # platforms keep the fenced code block. diff --git a/hermes_cli/session_export.py b/hermes_cli/session_export.py index 6f7baf81d0c89..71b66599e0d96 100644 --- a/hermes_cli/session_export.py +++ b/hermes_cli/session_export.py @@ -55,15 +55,24 @@ def render_sessions_export( export_format = normalize_export_format(fmt) export_only = normalize_export_only(only) + # Redact structured content BEFORE serialization (#20785 follow-up). + # A text pass alone misses credential values nested in structured + # messages: once JSONL-serialized, the key is escaped (\"client_secret\") + # so the JSON-field regex never fires, and opaque values have no + # recognizable prefix. Walking the structure first masks values under + # credential-named keys regardless of shape, and force=True keeps this + # egress boundary active even when security.redact_secrets is disabled. + from agent.redact import redact_structured + session_list = redact_structured(session_list) + if export_format == "jsonl": rendered = _render_jsonl(session_list, only=export_only) else: rendered = _render_markdown(session_list, only=export_only) - # Never let exported transcripts carry credential values (#20785). - # Redaction replaces matches with a plain placeholder, so JSON/MD - # structure stays intact. + # Belt-and-suspenders text pass over the serialized output so + # prose-embedded credentials (URLs, headers, KEY=value) are also masked. from agent.redact import redact_sensitive_text - return redact_sensitive_text(rendered) + return redact_sensitive_text(rendered, force=True) def export_record_count( diff --git a/tests/agent/test_redact.py b/tests/agent/test_redact.py index b52c7c1aed91b..457867e2b75ac 100644 --- a/tests/agent/test_redact.py +++ b/tests/agent/test_redact.py @@ -4,7 +4,7 @@ import pytest -from agent.redact import mask_secret, redact_cdp_url, redact_sensitive_text, RedactingFormatter +from agent.redact import mask_secret, redact_cdp_url, redact_structured, redact_sensitive_text, RedactingFormatter @pytest.fixture(autouse=True) @@ -1051,3 +1051,52 @@ def test_printable_mask_unchanged(self): def test_all_control_value_returns_empty_fallback(self): assert mask_secret("\n\x85\u200b") == "" assert mask_secret("\n\x85\u200b", empty="(not set)") == "(not set)" + + +class TestStructuredRedaction: + """redact_structured: deep-walk of JSON-like structures for egress + boundaries (session exports). Key context alone identifies opaque + credential values that no text pattern could match.""" + + def test_redacts_opaque_value_under_secret_key(self): + out = redact_structured( + {"client_secret": "opaque-client-secret-abc123", "client_id": "cid-9"} + ) + assert "opaque-client-secret-abc123" not in str(out) + assert out["client_secret"] != "opaque-client-secret-abc123" + assert out["client_id"] == "cid-9" # non-secret key untouched + + def test_redacts_nested_structures(self): + out = redact_structured( + { + "messages": [ + {"role": "tool", "content": {"apiKey": "sk-opaque-abc", "text": "hi"}}, + ] + } + ) + assert "sk-opaque-abc" not in str(out) + assert out["messages"][0]["content"]["text"] == "hi" + + def test_redacts_json_string_payloads(self): + import json as _json + + payload = _json.dumps({"client_secret": "opaque-xyz-789"}) + out = redact_structured({"content": payload}) + assert "opaque-xyz-789" not in str(out) + + def test_force_redacts_when_global_toggle_disabled(self, monkeypatch): + monkeypatch.setattr("agent.redact._REDACT_ENABLED", False) + out = redact_structured( + {"client_secret": "GOCSPX-abcdefghij1234567890", "note": "hello"} + ) + assert "GOCSPX-abcdefghij1234567890" not in str(out) + assert out["note"] == "hello" + + def test_leaves_non_secret_structure_unchanged(self): + value = { + "id": "sess-1", + "title": "Debug auth flow", + "message_count": 5, + "messages": [{"role": "user", "content": "Why is login broken?"}], + } + assert redact_structured(value) == value diff --git a/tests/hermes_cli/test_session_export.py b/tests/hermes_cli/test_session_export.py index 54e3f38040256..635ecda175bb7 100644 --- a/tests/hermes_cli/test_session_export.py +++ b/tests/hermes_cli/test_session_export.py @@ -95,6 +95,70 @@ def test_export_record_count_switches_unit_for_prompt_only_exports(): ) +def test_jsonl_export_redacts_embedded_client_secret_dict_value(): + """#20785 follow-up: opaque client_secret nested in structured message + content must not survive export. The key context alone identifies the + value as a secret, even though 'opaque-client-secret-...' has no + recognizable prefix and the serialized key is JSON-escaped.""" + opaque = "opaque-client-secret-abc123XYZ" + session = _sample_session() + session["messages"].append( + { + "id": 6, + "role": "tool", + "tool_name": "google_api", + "content": {"client_secret": opaque, "client_id": "cid-123"}, + "timestamp": 1700000005, + } + ) + rendered = render_sessions_export([session], fmt="jsonl") + assert opaque not in rendered + # JSONL stays parseable after redaction. + for line in rendered.strip().splitlines(): + json.loads(line) + + +def test_jsonl_export_redacts_embedded_client_secret_json_string(): + """Same leak through a JSON *string* payload (tool output that embeds a + credential blob as text).""" + opaque = "opaque-client-secret-abc123XYZ" + session = _sample_session() + session["messages"].append( + { + "id": 6, + "role": "tool", + "tool_name": "google_api", + "content": json.dumps({"client_secret": opaque}), + "timestamp": 1700000005, + } + ) + rendered = render_sessions_export([session], fmt="jsonl") + assert opaque not in rendered + for line in rendered.strip().splitlines(): + json.loads(line) + + +def test_export_redaction_forced_even_when_global_toggle_disabled(monkeypatch): + """Egress boundary must redact regardless of security.redact_secrets.""" + import agent.redact as redact_mod + + opaque = "GOCSPX-abcdefghij1234567890" + monkeypatch.setattr(redact_mod, "_REDACT_ENABLED", False) + session = _sample_session() + session["messages"].append( + { + "id": 6, + "role": "tool", + "tool_name": "google_api", + "content": {"client_secret": opaque}, + "timestamp": 1700000005, + } + ) + rendered = render_sessions_export([session], fmt="jsonl") + assert opaque not in rendered + assert "GOCSPX" not in rendered + + def test_sessions_export_cli_prompt_only_stdout(monkeypatch, capsys): import hermes_cli.main as main_mod import hermes_state