fix(redact): stop masking prose words that embed a secret keyword (port of ironclaw#6129)

agent/redact.py

@@ -174,6 +174,85 @@
     re.IGNORECASE | re.MULTILINE,
 )
 
+# Word-boundary validation for the mixed/lowercase key patterns above
+# (_CFG_DOTTED_RE, _CFG_ANCHORED_RE, _YAML_ASSIGN_RE).
+#
+# Those key classes allow arbitrary alphanumeric affixes around the secret
+# keyword so real key names like ``client_secret``, ``clientSecret``, and
+# ``s3.secret-key`` match. The side effect: ordinary prose/document words that
+# merely CONTAIN a keyword also matched — ``Secretary: J.Smith`` (secret),
+# ``tokenizer: cl100k_base`` (token), ``author=Smith`` (auth) — mangling
+# legitimate content on the surfaces that run these passes (browser snapshots,
+# log lines, kanban summaries, CLI-echoed command output). Ported from
+# nearai/ironclaw#6129, where the same substring false positive ("Secretary of
+# the Treasury" matching the ``secret`` marker) scrubbed legitimate tool
+# results from the replayed transcript and sent the model into a re-fetch
+# loop.
+#
+# A keyword occurrence only counts when it sits at a word boundary within the
+# key: at the key's edge, next to a non-letter (``_ - . 3``), or at a
+# camelCase transition (``clientSecret``, ``secretKey``, ``APIToken``). A
+# trailing plural ``s`` is treated as part of the keyword (``secrets:``,
+# ``tokens:``). Common concatenated compounds keep matching via explicit
+# alternatives (``authtoken`` ngrok, ``authkey`` tailscale, ``secretkey``
+# minio, ``apikey``). Embedded occurrences inside a larger word
+# (``secretary``, ``tokenizer``, ``authored``, ``credentialing``) no longer
+# match. ALL-CAPS keys keep the legacy embedded matching (``MYTOKEN=…``) — an
+# all-caps key is almost never prose, the same rationale as _ENV_ASSIGN_RE.
+_KEY_KEYWORD_RE = re.compile(
+    r"(?:api|auth|access|refresh|session|secret)[ _.\-]?(?:key|token)"
+    r"|token|secret|passwd|password|credential|auth",
+    re.IGNORECASE,
+)
+
+
+def _is_word_start(s: str, i: int) -> bool:
+    """True if position ``i`` in ``s`` begins a word (not mid-word)."""
+    if i == 0:
+        return True
+    prev, cur = s[i - 1], s[i]
+    if not prev.isalpha():
+        return True
+    if cur.isupper() and prev.islower():
+        return True  # camelCase: clientSecret
+    # Acronym run ending: APIToken — the 'T' begins a new word when it is
+    # followed by lowercase while the preceding run is uppercase.
+    if cur.isupper() and prev.isupper() and i + 1 < len(s) and s[i + 1].islower():
+        return True
+    return False
+
+
+def _is_word_end(s: str, j: int, *, allow_plural: bool = True) -> bool:
+    """True if position ``j`` (exclusive end) in ``s`` ends a word."""
+    if j >= len(s):
+        return True
+    cur = s[j]
+    if not cur.isalpha():
+        return True
+    if cur.isupper() and s[j - 1].islower():
+        return True  # camelCase continuation: secretKey
+    if allow_plural and cur in "sS":
+        return _is_word_end(s, j + 1, allow_plural=False)
+    return False
+
+
+def _key_has_secret_keyword(key: str) -> bool:
+    """True if ``key`` contains a secret keyword at a word boundary.
+
+    Post-match validator for _CFG_DOTTED_RE / _CFG_ANCHORED_RE /
+    _YAML_ASSIGN_RE hits — rejects prose words that merely embed a keyword
+    (``secretary``, ``tokenizer``, ``authored``). Safe to call with the
+    _ENV_ASSIGN_RE key too: all-caps keys short-circuit to the legacy
+    embedded-match behavior.
+    """
+    letters = [c for c in key if c.isalpha()]
+    if letters and all(c.isupper() for c in letters):
+        return True  # legacy all-caps behavior (MYTOKEN=…)
+    for m in _KEY_KEYWORD_RE.finditer(key):
+        if _is_word_start(key, m.start()) and _is_word_end(key, m.end()):
+            return True
+    return False
+
 # JSON field patterns: "apiKey": "value", "token": "value", etc.
 _JSON_KEY_NAMES = r"(?:api_?[Kk]ey|token|secret|password|access_token|refresh_token|auth_token|bearer|secret_value|raw_secret|secret_input|key_material)"
 _JSON_FIELD_RE = re.compile(
@@ -556,6 +635,13 @@ def _redact_env(m):
                 # prose/log contexts (issue #2852): ``KEY=os.getenv('X')``.
                 if _ENV_LOOKUP_VALUE_RE.match(value):
                     return m.group(0)
+                # Keyword must sit at a word boundary within the key —
+                # ``author=Smith`` / ``press.secretary=…`` are prose, not
+                # credentials (ported from nearai/ironclaw#6129). All-caps
+                # keys (the _ENV_ASSIGN_RE shape) short-circuit to legacy
+                # embedded matching inside the helper.
+                if not _key_has_secret_keyword(name):
+                    return m.group(0)
                 return f"{name}={quote}{_mask_token(value)}{quote}"
             text = _ENV_ASSIGN_RE.sub(_redact_env, text)
             # Lowercase/dotted config keys (issue #16413). Skip URLs entirely —
@@ -589,6 +675,11 @@ def _redact_yaml(m):
                 # not a leaked secret value.
                 if _ENV_LOOKUP_VALUE_RE.match(value):
                     return m.group(0)
+                # Keyword must sit at a word boundary within the key —
+                # ``Secretary: J.Smith`` / ``tokenizer: cl100k_base`` are
+                # document text, not credentials (nearai/ironclaw#6129).
+                if not _key_has_secret_keyword(key):
+                    return m.group(0)
                 return f"{key}{sep}{_mask_token(value)}"
             text = _YAML_ASSIGN_RE.sub(_redact_yaml, text)
 

tests/agent/test_redact.py

@@ -1046,3 +1046,96 @@ def test_non_string_input_coerced(self):
 
     def test_none_returns_empty(self):
         assert redact_cdp_url(None) == ""
+
+
+class TestKeywordWordBoundary:
+    """Ported from nearai/ironclaw#6129 — a secret keyword embedded inside a
+    larger prose word (``Secretary`` ⊃ ``secret``, ``tokenizer`` ⊃ ``token``,
+    ``authored`` ⊃ ``auth``) must NOT trigger the lowercase/dotted/YAML config
+    passes. Real key shapes (separators, camelCase, acronyms, plurals, common
+    concatenated compounds, all-caps env style) must keep redacting.
+    """
+
+    # ── prose words embedding a keyword are preserved ──────────────────
+
+    def test_secretary_yaml_value_preserved(self):
+        text = "Secretary: JanetYellen1234567890"
+        assert redact_sensitive_text(text) == text
+
+    def test_undersecretary_preserved(self):
+        text = "Undersecretary: RobertSmith123456789"
+        assert redact_sensitive_text(text) == text
+
+    def test_tokenizer_yaml_value_preserved(self):
+        # HuggingFace model-card style metadata.
+        text = "tokenizer: cl100k_base_long_name_x"
+        assert redact_sensitive_text(text) == text
+
+    def test_secretariat_preserved(self):
+        text = "secretariat: GenevaOffice123456789"
+        assert redact_sensitive_text(text) == text
+
+    def test_secretary_equals_assignment_preserved(self):
+        text = "secretary=JohnSmith12345678901234"
+        assert redact_sensitive_text(text) == text
+
+    def test_dotted_secretary_preserved(self):
+        text = "press.secretary=KarineJeanPierre123"
+        assert redact_sensitive_text(text) == text
+
+    def test_bibtex_author_assignment_preserved(self):
+        # ``author`` embeds the ``auth`` keyword — citation keys are prose.
+        text = "author=Smith2020LongCitationKey1"
+        assert redact_sensitive_text(text) == text
+
+    def test_credentialing_preserved(self):
+        text = "credentialing=enabled_long_value_12345"
+        assert redact_sensitive_text(text) == text
+
+    # ── real key shapes still redact ────────────────────────────────────
+
+    def test_separator_keys_still_redacted(self):
+        for text in (
+            "client_secret: abc123def456ghi789jkl",
+            "auth_token: xyz789xyz789xyz789xyz",
+            "my_secret: topvalue123456789012345",
+            "db.password=hunter2verylongpassword",
+        ):
+            result = redact_sensitive_text(text)
+            assert result != text, text
+
+    def test_camelcase_keys_still_redacted(self):
+        for text in (
+            "clientSecret: abc123def456ghi789jkl",
+            "secretKey: abc123def456ghi789jklmno",
+            "APIToken: abc123def456ghi789jklmn",
+        ):
+            result = redact_sensitive_text(text)
+            assert result != text, text
+
+    def test_concatenated_compounds_still_redacted(self):
+        # ngrok authtoken, tailscale authkey, minio secretkey, accesstoken.
+        for text in (
+            "authtoken: 2abcdefghij0123456789_ngrok",
+            "authkey=tskey-auth-abcdef123456789",
+            "secretkey: abc123def456ghi789jklmno",
+            "accesstoken: abcdefghij0123456789xyz",
+        ):
+            result = redact_sensitive_text(text)
+            assert result != text, text
+
+    def test_plural_keys_still_redacted(self):
+        text = "secrets: hunter2hunter2hunter2hh"
+        result = redact_sensitive_text(text)
+        assert "hunter2hunter2hunter2hh" not in result
+
+    def test_digit_boundary_still_redacted(self):
+        text = "oauth2_token: abcdefghij0123456789"
+        result = redact_sensitive_text(text)
+        assert "abcdefghij0123456789" not in result
+
+    def test_all_caps_embedded_keyword_still_redacted(self):
+        # All-caps keys keep legacy embedded matching (MYTOKEN=…).
+        text = "MYTOKEN=abcdefgh1234567890123456"
+        result = redact_sensitive_text(text)
+        assert "abcdefgh1234567890123456" not in result